Switch torquedata entirely to Django

As per discussion in issue #26 and over email correspondence, the plan
was to keep torquedata in Flask but use the Django ORM to interface with
postgres, keeping the option of migrating entirely to Django in the
future.

In practice, this presented some nontrivial issues. Since the data model
effectively *is* the application, torquedata would effectively become a
Django project that happens to use Flask to handle views and routing.
Support for non-standard Django projects online is minimal and
ironically requires more specific knowledge of Django's inner workings.
Torquedata's views are so dependent on the old pickling data model that
they will all have to be rewritten anyways, which means sticking with
Flask is an unnecessary complication.

A new Django app was created with `django-admin startproject`, and the
models and management commands were copied over.

Pare down Django boilerplate

Issue #26.

bleh

torquedata/.gitignore

@@ -1,2 +0,0 @@
-djangosettings.py
-flasksettings.py

torquedata/Pipfile

@@ -1,14 +1,12 @@
 [[source]]
-url = "https://pypi.python.org/simple"
-verify_ssl = true
 name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
 
 [packages]
-flask = "*"
-whoosh = "*"
 django = "*"
 
-[dev-packages]
-
 [requires]
 python_version = "3.7"

torquedata/core/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class CoreConfig(AppConfig):
+    name = 'core'

torquedata/core/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

torquedata/manage.py

@@ -1,8 +1,22 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
 import os
 import sys
 
-if __name__ == '__main__':
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djangosettings')
-    from django.core.management import execute_from_command_line
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'torquedata.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
     execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

torquedata/torquedata/__init__.py

@@ -1,151 +0,0 @@
-from flask import Flask
-
-import csv
-import configparser
-import os
-import pickle
-import json
-from whoosh import index
-from whoosh.index import create_in
-from whoosh.fields import *
-
-app = Flask(__name__)
-
-try:
-    app.config.from_object('config')
-except:
-    pass
-
-sheet_config = configparser.ConfigParser()
-sheet_config.read(os.path.join(app.config['SPREADSHEET_FOLDER'], "sheets"))
-
-try:
-    with open(os.path.join(app.config['SPREADSHEET_FOLDER'], "permissions"), 'rb') as f:
-        permissions = pickle.load(f)
-except Exception:
-    permissions = {}
-
-try:
-    with open(os.path.join(app.config['SPREADSHEET_FOLDER'], "templates"), 'rb') as f:
-        templates = pickle.load(f)
-except Exception:
-    templates = {}
-
-try:
-    with open(os.path.join(app.config['SPREADSHEET_FOLDER'], "attachment_config"), 'rb') as f:
-        attachment_config = pickle.load(f)
-except Exception:
-    attachment_config = {}
-
-try:
-    with open(os.path.join(app.config['SPREADSHEET_FOLDER'], "users"), 'rb') as f:
-        users = pickle.load(f)
-except Exception:
-    users = {}
-
-data = {}
-indices = {}
-def cull_invalid_columns(o, valid_fields):
-    return {k:v for (k,v) in o.items() if (k in valid_fields)}
-
-def cull_invalid_objects(group, sheet_name, wiki_key):
-    if sheet_name not in permissions.keys():
-        return []
-    if wiki_key not in permissions[sheet_name].keys():
-        return []
-    if group not in permissions[sheet_name][wiki_key].keys():
-        return []
-    elif "valid_ids" in permissions[sheet_name][wiki_key][group].keys():
-        valid_ids = permissions[sheet_name][wiki_key][group]["valid_ids"];
-        return [ o for o in data[sheet_name].values() if (o[sheet_config[sheet_name]["key_column"]] in valid_ids) ]
-    else:
-        return list(data[sheet_name].values())
-
-def permissions_sha(sheet_name, wiki_key, group):
-    import hashlib
-
-    return hashlib.sha224(
-        sheet_name.encode('utf-8') +
-        str(permissions[sheet_name][wiki_key][group]["valid_ids"]).encode('utf-8') +
-        str(permissions[sheet_name][wiki_key][group]["columns"]).encode('utf-8')
-    ).hexdigest()
-
-def index_search(group, sheet_name, wiki_key):
-    sha = permissions_sha(sheet_name, wiki_key, group)
-    dir = os.path.join(app.config['SPREADSHEET_FOLDER'], sheet_name, "indices", sha)
-
-    if(index.exists_in(dir)):
-        print("Index already exists for " + sheet_name + " / " + wiki_key + " / " + group + " (or comparable)")
-        ix = index.open_dir(dir)
-        indices[sha] = ix
-        return
-
-    try:
-        os.mkdir(dir)
-    except FileExistsError:
-        pass
-
-    print("Reindexing for " + sheet_name + " / " + group)
-    schema = Schema(key=ID(stored=True, unique=True), content=TEXT)
-    ix = create_in(dir, schema)
-    writer = ix.writer()
-    for o in cull_invalid_objects(group, sheet_name, wiki_key):
-        writer.add_document(
-                key=o[sheet_config[sheet_name]["key_column"]],
-                content=" ".join([str(c) for c in cull_invalid_columns(o, permissions[sheet_name][wiki_key][group]["columns"]).values()])
-                )
-    writer.commit()
-
-    indices[sha] = ix
-
-    return ""
-
-
-def load_sheet(sheet_name):
-    data[sheet_name] = {}
-    reader = csv.reader(
-            open(os.path.join(app.config.get("SPREADSHEET_FOLDER"), sheet_name, sheet_name + ".csv"), encoding='utf-8'),
-            delimiter=',',
-            quotechar='"'
-            )
-
-    if sheet_name not in templates:
-        templates[sheet_name] = {}
-
-    if sheet_name not in permissions:
-        permissions[sheet_name] = {}
-
-    header = next(reader)
-    column_types = next(reader)
-    for row in reader:
-        o = {}
-        for (field, column_type, cell) in zip(header, column_types, row):
-            if column_type == 'list':
-                # This may be reversed as a decision at some point, but the empty cell
-                # from the csv comes through as the empty string, meaning that the user
-                # probably wants the list to be empty as well.
-                if cell == '':
-                    cell = []
-                else:
-                    cell = cell.strip().split("\n")
-            elif column_type == 'json':
-                if cell == '':
-                    cell = {}
-                else:
-                    cell = json.loads(cell)
-            o[field] = cell
-        data[sheet_name][o[sheet_config[sheet_name]["key_column"]]] = o
-
-    for wiki_key in permissions[sheet_name].keys():
-        for group in permissions[sheet_name][wiki_key].keys():
-            sha = permissions_sha(sheet_name, wiki_key, group)
-            dir = os.path.join(app.config['SPREADSHEET_FOLDER'], sheet_name, wiki_key, "indices", sha)
-            index_search(group, sheet_name, wiki_key)
-
-for sheet_name in sheet_config.sections():
-    if sheet_name is "DEFAULT":
-        continue
-
-    load_sheet(sheet_name)
-
-from torquedata import routes

torquedata/torquedata/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for torquedata project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.1/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'torquedata.settings')
+
+application = get_asgi_application()