From fefcb928da85873a240c206a7e201724b1652a84 Mon Sep 17 00:00:00 2001
From: Alexander Leibzon <alexander.leibzon@gmail.com>
Date: Tue, 27 Sep 2016 19:13:44 +0300
Subject: [PATCH 1/4] add memsql as datasource
---
Vagrantfile | 2 +-
redash/models.py | 2 +-
redash/query_runner/memsql_ds.py | 147 +++++++++++++++++++++
redash/settings.py | 1 +
requirements.txt | 2 +
requirements_all_ds.txt | 1 +
setup/ubuntu/files/redash_supervisord_init | 0
7 files changed, 153 insertions(+), 2 deletions(-)
create mode 100644 redash/query_runner/memsql_ds.py
mode change 100644 => 100755 setup/ubuntu/files/redash_supervisord_init
diff --git a/Vagrantfile b/Vagrantfile
index 875cca8fa6..1611006f3d 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -10,6 +10,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.network "forwarded_port", guest: 5000, host: 9001
config.vm.provision "shell" do |s|
s.inline = "/opt/redash/current/setup/vagrant/provision.sh"
- s.privileged = false
+ s.privileged = true
end
end
diff --git a/redash/models.py b/redash/models.py
index c9ca984b9a..5df37508a5 100644
--- a/redash/models.py
+++ b/redash/models.py
@@ -1225,7 +1225,7 @@ def to_dict(self):
return d
-all_models = (Organization, Group, DataSource, DataSourceGroup, User, QueryResult, Query, Alert, Dashboard, Visualization, Widget, Event, NotificationDestination, AlertSubscription, ApiKey)
+all_models = (Organization, Group, DataSource, DataSourceGroup, User, QueryResult, Query, Alert, Dashboard, Visualization, Widget, Event, NotificationDestination, AlertSubscription, ApiKey, QuerySnippet)
def init_db():
diff --git a/redash/query_runner/memsql_ds.py b/redash/query_runner/memsql_ds.py
new file mode 100644
index 0000000000..14e7f4d119
--- /dev/null
+++ b/redash/query_runner/memsql_ds.py
@@ -0,0 +1,147 @@
+import json
+import logging
+import sys
+
+from redash.query_runner import *
+from redash.utils import JSONEncoder
+
+logger = logging.getLogger(__name__)
+
+try:
+ from memsql.common import database
+ enabled = True
+except ImportError, e:
+ logger.warning(e)
+ enabled = False
+
+COLUMN_NAME = 0
+COLUMN_TYPE = 1
+
+types_map = {
+ 'BIGINT': TYPE_INTEGER,
+ 'TINYINT': TYPE_INTEGER,
+ 'SMALLINT': TYPE_INTEGER,
+ 'MEDIUMINT': TYPE_INTEGER,
+ 'INT': TYPE_INTEGER,
+ 'DOUBLE': TYPE_FLOAT,
+ 'DECIMAL': TYPE_FLOAT,
+ 'FLOAT': TYPE_FLOAT,
+ 'REAL': TYPE_FLOAT,
+ 'BOOL': TYPE_BOOLEAN,
+ 'BOOLEAN': TYPE_BOOLEAN,
+ 'TIMESTAMP': TYPE_DATETIME,
+ 'DATETIME': TYPE_DATETIME,
+ 'DATE': TYPE_DATETIME,
+ 'JSON': TYPE_STRING,
+ 'CHAR': TYPE_STRING,
+ 'VARCHAR': TYPE_STRING
+}
+
+
+class MemSQL(BaseSQLQueryRunner):
+ @classmethod
+ def configuration_schema(cls):
+ return {
+ "type": "object",
+ "properties": {
+ "host": {
+ "type": "string"
+ },
+ "port": {
+ "type": "number"
+ },
+ "user": {
+ "type": "string"
+ },
+ "password": {
+ "type": "string"
+ }
+
+ },
+ "required": ["host", "port"]
+ }
+
+ @classmethod
+ def annotate_query(cls):
+ return False
+
+ @classmethod
+ def type(cls):
+ return "memsql"
+
+ @classmethod
+ def enabled(cls):
+ return enabled
+
+ def __init__(self, configuration):
+ super(MemSQL, self).__init__(configuration)
+
+ def _get_tables(self, schema):
+ try:
+ schemas_query = "show schemas"
+
+ tables_query = "show tables in %s"
+
+ columns_query = "show columns in %s"
+
+ for schema_name in filter(lambda a: len(a) > 0, map(lambda a: str(a['Database']), self._run_query_internal(schemas_query))):
+ for table_name in filter(lambda a: len(a) > 0, map(lambda a: str(a['Tables_in_%s' % schema_name]), self._run_query_internal(tables_query % schema_name))):
+ columns = filter(lambda a: len(a) > 0, map(lambda a: str(a['Field']), self._run_query_internal(columns_query % table_name)))
+
+ schema[table_name] = {'name': table_name, 'columns': columns}
+ except Exception, e:
+ raise sys.exc_info()[1], None, sys.exc_info()[2]
+ return schema.values()
+
+ def run_query(self, query):
+
+ cursor = None
+ try:
+ cursor = database.connect(**self.configuration.to_dict())
+
+ res = cursor.query(query)
+ # column_names = []
+ # columns = []
+ #
+ # for column in cursor.description:
+ # column_name = column[COLUMN_NAME]
+ # column_names.append(column_name)
+ #
+ # columns.append({
+ # 'name': column_name,
+ # 'friendly_name': column_name,
+ # 'type': types_map.get(column[COLUMN_TYPE], None)
+ # })
+
+ rows = [dict(zip(list(row.keys()), list(row.values()))) for row in res]
+
+
+ #====================================================================================================
+ #temporary - until https://github.com/memsql/memsql-python/pull/8 gets merged
+ #====================================================================================================
+ columns = []
+ column_names = rows[0].keys() if rows else None
+ for column in column_names:
+ columns.append({
+ 'name': column,
+ 'friendly_name': column,
+ 'type': None
+ })
+
+ data = {'columns': columns, 'rows': rows}
+ json_data = json.dumps(data, cls=JSONEncoder)
+ error = None
+ except KeyboardInterrupt:
+ cursor.close()
+ error = "Query cancelled by user."
+ json_data = None
+ except Exception as e:
+ logging.exception(e)
+ raise sys.exc_info()[1], None, sys.exc_info()[2]
+ finally:
+ if cursor:
+ cursor.close()
+
+ return json_data, error
+
+register(MemSQL)
diff --git a/redash/settings.py b/redash/settings.py
index 9725d9164f..5b1454df48 100644
--- a/redash/settings.py
+++ b/redash/settings.py
@@ -175,6 +175,7 @@ def all_settings():
'redash.query_runner.sqlite',
'redash.query_runner.dynamodb_sql',
'redash.query_runner.mssql',
+ 'redash.query_runner.memsql_ds',
]
enabled_query_runners = array_from_string(os.environ.get("REDASH_ENABLED_QUERY_RUNNERS", ",".join(default_query_runners)))
diff --git a/requirements.txt b/requirements.txt
index 0a348ea924..d46a454d3c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -40,3 +40,5 @@ xlsxwriter==0.8.4
pystache==0.5.4
parsedatetime==2.1
cryptography==1.4
+oauthlib==2.0.0
+WTForms==2.1
diff --git a/requirements_all_ds.txt b/requirements_all_ds.txt
index 674567e3b0..5b16cbaca8 100644
--- a/requirements_all_ds.txt
+++ b/requirements_all_ds.txt
@@ -17,3 +17,4 @@ sasl>=0.1.3
thrift>=0.8.0
thrift_sasl>=0.1.0
cassandra-driver==3.1.1
+memsql==2.16.0
diff --git a/setup/ubuntu/files/redash_supervisord_init b/setup/ubuntu/files/redash_supervisord_init
old mode 100644
new mode 100755
From 79187cd29a57ba4c5c90e9e0c67aa61b966540f8 Mon Sep 17 00:00:00 2001
From: Alexander Leibzon <alexander.leibzon@gmail.com>
Date: Tue, 2 May 2017 23:36:11 +0300
Subject: [PATCH 2/4] get_schema fix
---
redash/query_runner/memsql_ds.py | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git a/redash/query_runner/memsql_ds.py b/redash/query_runner/memsql_ds.py
index 14e7f4d119..ed2dfdad3f 100644
--- a/redash/query_runner/memsql_ds.py
+++ b/redash/query_runner/memsql_ds.py
@@ -39,6 +39,8 @@
class MemSQL(BaseSQLQueryRunner):
+ noop_query = 'SELECT 1'
+
@classmethod
def configuration_schema(cls):
return {
@@ -86,6 +88,7 @@ def _get_tables(self, schema):
for schema_name in filter(lambda a: len(a) > 0, map(lambda a: str(a['Database']), self._run_query_internal(schemas_query))):
for table_name in filter(lambda a: len(a) > 0, map(lambda a: str(a['Tables_in_%s' % schema_name]), self._run_query_internal(tables_query % schema_name))):
+ table_name = '.'.join((schema_name, table_name))
columns = filter(lambda a: len(a) > 0, map(lambda a: str(a['Field']), self._run_query_internal(columns_query % table_name)))
schema[table_name] = {'name': table_name, 'columns': columns}
@@ -93,7 +96,7 @@ def _get_tables(self, schema):
raise sys.exc_info()[1], None, sys.exc_info()[2]
return schema.values()
- def run_query(self, query):
+ def run_query(self, query, user):
cursor = None
try:
@@ -121,12 +124,14 @@ def run_query(self, query):
#====================================================================================================
columns = []
column_names = rows[0].keys() if rows else None
- for column in column_names:
- columns.append({
- 'name': column,
- 'friendly_name': column,
- 'type': None
- })
+
+ if column_names:
+ for column in column_names:
+ columns.append({
+ 'name': column,
+ 'friendly_name': column,
+ 'type': None
+ })
data = {'columns': columns, 'rows': rows}
json_data = json.dumps(data, cls=JSONEncoder)
From b9f8b6cdbf4fa1cc49865764d2e5b9007e64f885 Mon Sep 17 00:00:00 2001
From: Alexander Leibzon <alexander.leibzon@gmail.com>
Date: Tue, 2 May 2017 23:49:09 +0300
Subject: [PATCH 3/4] reformat, as for pep-8
---
redash/query_runner/memsql_ds.py | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git a/redash/query_runner/memsql_ds.py b/redash/query_runner/memsql_ds.py
index ed2dfdad3f..ddba85a320 100644
--- a/redash/query_runner/memsql_ds.py
+++ b/redash/query_runner/memsql_ds.py
@@ -9,6 +9,7 @@
try:
from memsql.common import database
+
enabled = True
except ImportError, e:
logger.warning(e)
@@ -86,10 +87,14 @@ def _get_tables(self, schema):
columns_query = "show columns in %s"
- for schema_name in filter(lambda a: len(a) > 0, map(lambda a: str(a['Database']), self._run_query_internal(schemas_query))):
- for table_name in filter(lambda a: len(a) > 0, map(lambda a: str(a['Tables_in_%s' % schema_name]), self._run_query_internal(tables_query % schema_name))):
+ for schema_name in filter(lambda a: len(a) > 0,
+ map(lambda a: str(a['Database']), self._run_query_internal(schemas_query))):
+ for table_name in filter(lambda a: len(a) > 0, map(lambda a: str(a['Tables_in_%s' % schema_name]),
+ self._run_query_internal(
+ tables_query % schema_name))):
table_name = '.'.join((schema_name, table_name))
- columns = filter(lambda a: len(a) > 0, map(lambda a: str(a['Field']), self._run_query_internal(columns_query % table_name)))
+ columns = filter(lambda a: len(a) > 0, map(lambda a: str(a['Field']),
+ self._run_query_internal(columns_query % table_name)))
schema[table_name] = {'name': table_name, 'columns': columns}
except Exception, e:
@@ -118,10 +123,9 @@ def run_query(self, query, user):
rows = [dict(zip(list(row.keys()), list(row.values()))) for row in res]
-
- #====================================================================================================
- #temporary - until https://github.com/memsql/memsql-python/pull/8 gets merged
- #====================================================================================================
+ # ====================================================================================================
+ # temporary - until https://github.com/memsql/memsql-python/pull/8 gets merged
+ # ====================================================================================================
columns = []
column_names = rows[0].keys() if rows else None
@@ -149,4 +153,5 @@ def run_query(self, query, user):
return json_data, error
+
register(MemSQL)
From 6b7234c910410d8bb90b8c1b9f95f298b928d775 Mon Sep 17 00:00:00 2001
From: Alexander Leibzon <alexander.leibzon@gmail.com>
Date: Thu, 18 May 2017 14:00:13 +0300
Subject: [PATCH 4/4] fixes
---
redash/query_runner/memsql_ds.py | 32 +++++++++++++++-----------------
requirements.txt | 2 --
2 files changed, 15 insertions(+), 19 deletions(-)
diff --git a/redash/query_runner/memsql_ds.py b/redash/query_runner/memsql_ds.py
index ddba85a320..d0dd3735a4 100644
--- a/redash/query_runner/memsql_ds.py
+++ b/redash/query_runner/memsql_ds.py
@@ -61,7 +61,8 @@ def configuration_schema(cls):
}
},
- "required": ["host", "port"]
+ "required": ["host", "port"],
+ "secret": ["password"]
}
@classmethod
@@ -80,25 +81,22 @@ def __init__(self, configuration):
super(MemSQL, self).__init__(configuration)
def _get_tables(self, schema):
- try:
- schemas_query = "show schemas"
+ schemas_query = "show schemas"
- tables_query = "show tables in %s"
+ tables_query = "show tables in %s"
- columns_query = "show columns in %s"
+ columns_query = "show columns in %s"
- for schema_name in filter(lambda a: len(a) > 0,
- map(lambda a: str(a['Database']), self._run_query_internal(schemas_query))):
- for table_name in filter(lambda a: len(a) > 0, map(lambda a: str(a['Tables_in_%s' % schema_name]),
- self._run_query_internal(
- tables_query % schema_name))):
- table_name = '.'.join((schema_name, table_name))
- columns = filter(lambda a: len(a) > 0, map(lambda a: str(a['Field']),
- self._run_query_internal(columns_query % table_name)))
+ for schema_name in filter(lambda a: len(a) > 0,
+ map(lambda a: str(a['Database']), self._run_query_internal(schemas_query))):
+ for table_name in filter(lambda a: len(a) > 0, map(lambda a: str(a['Tables_in_%s' % schema_name]),
+ self._run_query_internal(
+ tables_query % schema_name))):
+ table_name = '.'.join((schema_name, table_name))
+ columns = filter(lambda a: len(a) > 0, map(lambda a: str(a['Field']),
+ self._run_query_internal(columns_query % table_name)))
- schema[table_name] = {'name': table_name, 'columns': columns}
- except Exception, e:
- raise sys.exc_info()[1], None, sys.exc_info()[2]
+ schema[table_name] = {'name': table_name, 'columns': columns}
return schema.values()
def run_query(self, query, user):
@@ -134,7 +132,7 @@ def run_query(self, query, user):
columns.append({
'name': column,
'friendly_name': column,
- 'type': None
+ 'type': TYPE_STRING
})
data = {'columns': columns, 'rows': rows}
diff --git a/requirements.txt b/requirements.txt
index 8fac8920f6..b703b559c4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -42,6 +42,4 @@ xlsxwriter==0.9.3
pystache==0.5.4
parsedatetime==2.1
cryptography==1.4
-oauthlib==2.0.0
-WTForms==2.1
simplejson==3.10.0