Import fix (#48)

* Fixed CSV import

* Ignoring editor files

* Added permission check on csv import/export

* Adding code coverage

* Added codecov badge

* Fixed coverage command

* Fixing coverage tests

* Passing commit sha to codecov

* Fixing test build

* Fixed line endings

* Reverted tests

.gitignore

@@ -0,0 +1,2 @@
+.vscode/
+database.db

README.md

@@ -2,6 +2,7 @@
 
 [![Copr build status](https://copr.fedorainfracloud.org/coprs/bitbyt3r/Tuber/package/tuber/status_image/last_build.png)](https://copr.fedorainfracloud.org/coprs/bitbyt3r/Tuber/package/tuber/)
 [![Heroku CI Status](https://tuber-ci-badge.herokuapp.com/last.svg)](https://dashboard.heroku.com/pipelines/6ebd065d-db02-419d-80bd-6406f271d992/tests)
+[![codecov](https://codecov.io/gh/magfest/tuber/branch/master/graph/badge.svg)](https://codecov.io/gh/magfest/tuber)
 
 Table of Contents
 =================

backend/.gitignore

@@ -1,6 +1,8 @@
 .DS_Store
 node_modules
 /dist
+.coverage
+coverage.xml
 
 tuber.json
 database.db

backend/pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
 testpaths=tests
 filterwarnings =
-    ignore::DeprecationWarning
+    ignore::DeprecationWarning

backend/requirements-test.txt

@@ -1,2 +1,3 @@
 .
-pytest
+pytest
+coverage

backend/tests/util.py → backend/tests/conftest.py

@@ -3,10 +3,12 @@
 import json
 import os
 
-os.environ['DATABASE_URL'] = "sqlite:///:memory:"
 import tuber
-tuber.migrate()
-tuber.app.config['TESTING'] = True
+settings_override = {
+        'TESTING': True,
+        'SQLALCHEMY_DATABASE_URI': "sqlite:///:memory:"
+    }
+tuber.app.config.update(settings_override)
 
 def csrf(client):
     for cookie in client.cookie_jar:
@@ -42,28 +44,37 @@ def get(*args, **kwargs):
             return rv
         _post = client.post
         def post(*args, **kwargs):
-            if not 'json' in kwargs:
-                kwargs['json'] = {}
-            kwargs['json']['csrf_token'] = csrf(client)
+            if 'data' in kwargs:
+                kwargs['data']['csrf_token'] = csrf(client)
+            else:
+                if not 'json' in kwargs:
+                    kwargs['json'] = {}
+                kwargs['json']['csrf_token'] = csrf(client)
             rv = _post(*args, **kwargs)
             return rv
         _patch = client.patch
         def patch(*args, **kwargs):
-            if not 'json' in kwargs:
-                kwargs['json'] = {}
-            kwargs['json']['csrf_token'] = csrf(client)
+            if 'data' in kwargs:
+                kwargs['data']['csrf_token'] = csrf(client)
+            else:
+                if not 'json' in kwargs:
+                    kwargs['json'] = {}
+                kwargs['json']['csrf_token'] = csrf(client)
             rv = _patch(*args, **kwargs)
             return rv
         _delete = client.delete
         def delete(*args, **kwargs):
-            if not 'json' in kwargs:
-                kwargs['json'] = {}
-            kwargs['json']['csrf_token'] = csrf(client)
+            if 'data' in kwargs:
+                kwargs['data']['csrf_token'] = csrf(client)
+            else:
+                if not 'json' in kwargs:
+                    kwargs['json'] = {}
+                kwargs['json']['csrf_token'] = csrf(client)
             rv = _delete(*args, **kwargs)
             return rv
         client.get = get
         client.post = post
         client.patch = patch
         client.delete = delete
         yield client
-    tuber.db.drop_all()
+    tuber.db.drop_all()

backend/tests/test_hotel.py

@@ -1,5 +1,4 @@
 from unittest.mock import patch
-from util import *
 import json
 
 @patch('tuber.api.hotels.requests.post')

backend/tests/test_importer.py

@@ -0,0 +1,48 @@
+import datetime
+import json
+import io
+
+def test_csv_export(client):
+    """Try exporting a table and make sure it matches the expected contents."""
+    badge = client.post("/api/events/1/badges", json={
+        "legal_name": "Test User",
+        "departments": []
+    }).json
+    assert(badge['legal_name'] == "Test User")
+
+    export = client.get("/api/importer/csv", query_string={"csv_type": "Badge"}).data.decode('UTF-8').strip()
+    assert(len(export.split("\n")) == 2)
+    header, exported_badge = export.split("\n")
+    header = header.split(",")
+    exported_badge = exported_badge.split(",")
+    badge_dict = {k:v for k,v in zip(header, exported_badge)}
+    assert(badge_dict['legal_name'] == "Test User")
+
+def test_csv_import(client):
+    client.post("/api/importer/csv", content_type="multipart/form-data", data={
+        "csv_type": "Badge",
+        "raw_import": False,
+        "full_import": False,
+        "files": (io.BytesIO(b"legal_name,event\nTest User 1,1"), 'badge.csv')
+    })
+    badges = client.get("/api/events/1/badges").json
+    assert(len(badges) == 1)
+
+    client.post("/api/importer/csv", content_type="multipart/form-data", data={
+        "csv_type": "Badge",
+        "raw_import": False,
+        "full_import": False,
+        "files": (io.BytesIO(b"legal_name,event\nTest User 2,1"), 'badge.csv')
+    })
+    badges = client.get("/api/events/1/badges").json
+    assert(len(badges) == 2)
+
+    client.post("/api/importer/csv", content_type="multipart/form-data", data={
+        "csv_type": "Badge",
+        "raw_import": False,
+        "full_import": True,
+        "files": (io.BytesIO(b"legal_name,event\nTest User 3,1"), 'badge.csv')
+    })
+    badges = client.get("/api/events/1/badges", query_string={"full": True}).json
+    assert(len(badges) == 1)
+    assert(badges[0]['legal_name'] == "Test User 3")

backend/tests/test_shifts.py

@@ -1,4 +1,3 @@
-from util import *
 import datetime
 import json
 

backend/tests/test_tuber.py

@@ -1,4 +1,4 @@
-from util import *
+from conftest import csrf
 import json
 
 def test_csrf(client_fresh):

backend/tuber/api/importer.py

@@ -1,10 +1,11 @@
 from tuber import app, config, db
-from flask import send_from_directory, send_file, request, jsonify
+from flask import send_from_directory, send_file, request, jsonify, Response
 from tuber.models import *
 from tuber.permissions import *
 from tuber.worker import worker_conn
 from sqlalchemy import or_
 from rq import Queue
+import sqlalchemy
 import requests
 import datetime
 import random
@@ -13,6 +14,63 @@
 import csv
 import io
 
+@app.route("/api/importer/csv", methods=["GET", "POST"])
+def csv_import():
+    if request.method == "GET":
+        if not check_permission("export.csv"):
+            return "Permission Denied", 403
+        export_type = request.args['csv_type']
+        model = globals()[export_type]
+        data = db.session.query(model).all()
+        cols = model.__table__.columns.keys()
+        def generate():
+            yield ','.join(cols)+"\n"
+            for row in data:
+                yield ','.join([str(getattr(row, x)) for x in cols])+"\n"
+        response = Response(generate(), mimetype="text/csv")
+        response.headers.set('Content-Disposition', 'attachment; filename={}.csv'.format(export_type))
+        return response
+    elif request.method == "POST":
+        if not check_permission("import.csv"):
+            return "Permission Denied", 403
+        import_type = request.form['csv_type']
+        model = globals()[import_type]
+        raw_import = request.form['raw_import'].lower().strip() == "true"
+        full_import = request.form['full_import'].lower().strip() == "true"
+        file = request.files['files']
+        data = file.read().decode('UTF-8').replace("\r\n", "\n")
+        if full_import:
+            db.session.query(model).delete()
+        rows = data.split("\n")
+        cols = rows[0].split(",")
+        rows = rows[1:]
+        def convert(key, val):
+            col = model.__table__.columns[key]
+            if col.nullable:
+                if val == 'None':
+                    return None
+            coltype = type(col.type)
+            if coltype is sqlalchemy.sql.sqltypes.Integer:
+                if val == '':
+                    return None
+                return int(val)
+            if coltype is sqlalchemy.sql.sqltypes.Boolean:
+                if val.lower() == "true":
+                    return True
+                return False
+            return val
+
+        count = 0
+        for row in rows:
+            if not row.strip():
+                continue
+            row = row.split(",")
+            new = model(**{key: convert(key, val) for key,val in zip(cols, row)})
+            db.session.add(new)
+            count += 1
+        db.session.commit()
+        return str(count), 200
+
 def get_uber_csv(session, model, url):
     data = session.post(url+"/devtools/export_model", data={"selected_model": model}).text
     stream = io.StringIO(data)

backend/tuber/csrf.py

@@ -22,7 +22,15 @@ def validate_csrf():
             g.data = dict(request.json)
             del g.data['csrf_token']
             return
-        return f"{request.method} Method requires json."
+        if not request.form is None:
+            if not 'csrf_token' in request.form:
+                return f"You must pass a csrf token in the body with all {request.method} requests that include a csrf cookie."
+            if request.form['csrf_token'] != request.cookies.get('csrf_token'):
+                return "Invalid csrf token."
+            g.data = dict(request.form)
+            del g.data['csrf_token']
+            return
+        return f"{request.method} Method requires json or form data."
 
 @app.after_request
 def insert_csrf(response):

backend/tuber/permissions.py

@@ -44,8 +44,12 @@ def get_user():
                     if "event" in request.args:
                         event = request.args['event']
                 if request.method == "POST":
-                    if "event" in request.json:
-                        event = request.json['event']
+                    if request.json:
+                        data = request.json
+                    else:
+                        data = request.form
+                    if "event" in data:
+                        event = int(data['event'])
                 g.event = event
                 g.badge = db.session.query(Badge).filter(Badge.user == g.user.id, Badge.event == event).one_or_none()
             else:

contrib/build-tests.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
 pip install pytest
+pip install coverage
 cd frontend
 npm install --only=dev

contrib/run-tests.sh

@@ -3,7 +3,8 @@
 set -e
 
 cd backend
-pytest
+coverage run --source=tuber -m pytest
+bash <(curl -s https://codecov.io/bash) -C $HEROKU_TEST_RUN_COMMIT_VERSION
 
 cd ../frontend
 npm run test

frontend/src/mixins/rest.js

@@ -66,6 +66,35 @@ function post(url, data) {
   return restFetch('POST', url, data);
 }
 
+function download(url, data) {
+  if (data === undefined) {
+    data = {};
+  }
+  data.csrf_token = window.$cookies.get('csrf_token');
+  const queryString = `?${Object.keys(data).map((key) => `${encodeURIComponent(key)}=${encodeURIComponent(data[key])}`).join('&')}`;
+  url += queryString;
+  window.location.href = url;
+}
+
+function upload(url, data) {
+  if (data === undefined) {
+    data = {};
+  }
+  data.csrf_token = window.$cookies.get('csrf_token');
+  const form = new FormData();
+  const keys = Object.keys(data);
+  for (let i = 0; i < keys.length; i += 1) {
+    form.append(keys[i], data[keys[i]]);
+  }
+  return fetch(url, {
+    method: 'POST',
+    headers: {
+    },
+    body: form,
+    credentials: 'include',
+  }).then((response) => response.json());
+}
+
 function list(endpoint) {
   const url = schema[endpoint].url.replace('<event>', this.$store.getters.event.id);
   return restFetch('GET', url);
@@ -185,10 +214,10 @@ function mapAsyncObjects(endpoints) {
 
 Vue.mixin({
   methods: {
-    restFetch, mapAsync, mapAsyncObjects, save, remove, load, dump, list, get, mapAsyncDump, post,
+    restFetch, mapAsync, mapAsyncObjects, save, remove, load, dump, list, get, mapAsyncDump, post, download, upload,
   },
 });
 
 export {
-  restFetch, mapAsync, mapAsyncObjects, save, remove, load, dump, list, get, mapAsyncDump, post,
+  restFetch, mapAsync, mapAsyncObjects, save, remove, load, dump, list, get, mapAsyncDump, post, download, upload,
 };