V2.2.0 add admin panel (#72)

* ADD user admin panel page

* CHG gitignore eslintcache

* ADD api testing utilities

* ADD mindebug target

* add admin panel menu options

* CHG move more things to useEffect and add assignments page

* ADD stats pages

* FIX some weird grid placements

* CHG ide dialog and more consistent spacing

* ADD theia page on admin panel, more consistent spacing and theia admin image

* CHG fix theia image tags

* ADD theia options, and kube working

* ADD stuff from KUBE DAY

* CHG bug fixes for testing phase

* CHG improve debug provisioning

* FIX join issue

* CHG strengthen join code validation

* CHG improve webhook username guess

* FIX safari issue, and add minikube support on MacOS

* CHG docker-compose commands pull new base images

* FIX theia session ide reaping id issue

* CHG theia session hostnames shorter

* CHG expand webhook tests a bit

Co-authored-by: Sejinkonye <se1346@nyu.edu>
Co-authored-by: John McCann Cunniff Jr <john.cunniff@voladynamics.com>

Makefile

@@ -1,9 +1,10 @@
-PERSISTENT_SERVICES := db traefik kibana elasticsearch-coordinating redis-master logstash adminer
-RESTART_ALWAYS_SERVICES := api web
-PUSH_SERVICES := api web logstash
+PERSISTENT_SERVICES := db traefik kibana elasticsearch-coordinating redis-master logstash adminer 
+RESTART_ALWAYS_SERVICES := api web rpc-worker
+PUSH_SERVICES := api web logstash theia-init theia-proxy theia-admin theia-xv6
 BUILD_ALLWAYS := api web
 
 
+
 CURRENT_DIR := $(shell basename $$(pwd) | tr '[:upper:]' '[:lower:]')
 IMAGES := $(shell \
 	ruby -ryaml -rjson -e 'puts JSON.pretty_generate(YAML.load(ARGF))' \
@@ -33,10 +34,11 @@ help:
 
 .PHONY: build        # Build all docker images
 build:
-	docker-compose build --parallel $(BUILD_ALLWAYS)
+	docker-compose build --parallel --pull $(BUILD_ALLWAYS)
 
 .PHONY: push         # Push images to registry.osiris.services (requires vpn)
-push: build
+push:
+	docker-compose build --parallel --pull $(PUSH_SERVICES)
 	docker-compose push $(PUSH_SERVICES)
 
 .PHONY: debug        # Start the cluster in debug mode
@@ -45,13 +47,33 @@ debug: build
 	docker-compose up \
 		-d --force-recreate \
 		$(RESTART_ALWAYS_SERVICES)
+	@echo 'Waiting a moment before running migrations'
+	sleep 3
+	@echo 'running migrations'
+	make -C api migrations
+	@echo 'seed: http://localhost/api/admin/seed/'
+	@echo 'auth: http://localhost/api/admin/auth/token/jmc1283'
+	@echo 'site: http://localhost/'
+
+.PHONY: mindebug     # Start the minimal cluster in debug mode
+mindebug: build
+	docker-compose up -d traefik db redis-master logstash
+	docker-compose up \
+		-d --force-recreate \
+		api web rpc-worker
+	@echo 'Waiting a moment before running migrations'
+	sleep 3
+	@echo 'running migrations'
+	make -C api migrations
+	@echo 'seed: http://localhost/api/admin/seed/'
+	@echo 'auth: http://localhost/api/admin/auth/token/jmc1283'
+	@echo 'site: http://localhost/'
 
 
 .PHONY: jupyter      # Start he jupyterhub container
 jupyter:
 	docker-compose up --force-recreate --build api-dev
 
-
 .PHONY: deploy       # Start the cluster in production mode
 deploy: check build restart
 

api/Makefile

@@ -9,15 +9,22 @@ LINT_FILES := $(shell find $(LINT_DIRECTORIES) -name '*.py' | xargs)
 all: venv
 
 venv:
-	virtualenv -p `which python3.8` venv
+	virtualenv -p `which python3` venv
 	./venv/bin/pip install -r ./requirements.txt -r ./requirements-dev.txt
 
+debug:
+	make -C .. debug
+
 yeetdb:
 	make -C .. yeetdb
 
 cleanyeetdb:
 	make -C .. cleanyeetdb
 
+.PHONY: migrations       # Run alembic migrations
+migrations: venv
+	./venv/bin/alembic upgrade head
+
 .PHONY: lint             # Run autopep8 then black on lint directories
 lint: venv
 	@echo 'autopep to check and fix un-python things'

api/anubis/app.py

@@ -39,15 +39,15 @@ def create_app():
 
     :return: Flask app
     """
-    from anubis.config import config
+    from anubis.config import Config
 
     # Import views
     from anubis.views.public import register_public_views
     from anubis.views.admin import register_admin_views
 
     # Create app
     app = Flask(__name__)
-    app.config.from_object(config)
+    app.config.from_object(Config())
 
     # Initialize app with all the extra services
     init_services(app)
@@ -69,7 +69,7 @@ def create_pipeline_app():
     :return: Flask app
     """
     from anubis.config import config
-    from anubis.views.pipeline.pipeline import pipeline
+    from anubis.views.pipeline import register_pipeline_views
 
     # Create app
     app = Flask(__name__)
@@ -79,6 +79,6 @@ def create_pipeline_app():
     init_services(app)
 
     # register blueprints
-    app.register_blueprint(pipeline)
+    register_pipeline_views(app)
 
     return app

api/anubis/config.py

@@ -1,12 +1,11 @@
 import logging
 import os
-import hashlib
+from datetime import timedelta
 
 
 class Config:
-    SECRET_KEY = os.environ.get(
-        "SECRET_KEY", default=hashlib.sha512(os.urandom(10)).hexdigest()
-    )
+    SECRET_KEY = None
+    STATS_REAP_DURATION = timedelta(days=60)
 
     # sqlalchemy
     SQLALCHEMY_DATABASE_URI = None
@@ -28,9 +27,13 @@ class Config:
 
     # Theia config
     THEIA_DOMAIN = ""
+    THEIA_TIMEOUT = timedelta(hours=6)
 
     def __init__(self):
         self.DEBUG = os.environ.get("DEBUG", default="0") == "1"
+
+        self.SECRET_KEY = os.environ.get("SECRET_KEY", default="DEBUG")
+
         self.SQLALCHEMY_DATABASE_URI = os.environ.get(
             "DATABASE_URI",
             default="mysql+pymysql://anubis:anubis@{}/anubis".format(
@@ -46,7 +49,13 @@ def __init__(self):
         )
 
         # Redis
-        self.CACHE_REDIS_HOST = os.environ.get("CACHE_REDIS_HOST", default="redis")
+        self.CACHE_REDIS_HOST = os.environ.get(
+            "CACHE_REDIS_HOST", default="redis-master"
+        )
+
+        self.CACHE_REDIS_PASSWORD = os.environ.get(
+            "REDIS_PASS", default="anubis"
+        )
 
         # Logger
         self.LOGGER_NAME = os.environ.get("LOGGER_NAME", default="anubis-api")
@@ -55,6 +64,7 @@ def __init__(self):
         self.THEIA_DOMAIN = os.environ.get(
             "THEIA_DOMAIN", default="ide.anubis.osiris.services"
         )
+        self.THEIA_TIMEOUT = timedelta(hours=6) if not self.DEBUG else timedelta(minutes=2)
 
         logging.info(
             "Starting with DATABASE_URI: {}".format(self.SQLALCHEMY_DATABASE_URI)

api/anubis/models/__init__.py

@@ -1,7 +1,7 @@
 import base64
 import logging
 import os
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from flask_sqlalchemy import SQLAlchemy
 from sqlalchemy_json import MutableJson
@@ -11,8 +11,10 @@
 db = SQLAlchemy()
 
 
-def default_id() -> db.Column:
-    return db.Column(db.String(128), primary_key=True, default=rand)
+def default_id(max_len=None) -> db.Column:
+    return db.Column(
+        db.String(128), primary_key=True, default=lambda: rand(max_len or 32)
+    )
 
 
 class Config(db.Model):
@@ -40,9 +42,6 @@ class User(db.Model):
     created = db.Column(db.DateTime, default=datetime.now)
     last_updated = db.Column(db.DateTime, default=datetime.now, onupdate=datetime.now)
 
-    repos = db.relationship("AssignmentRepo", cascade="all,delete")
-    in_class = db.relationship("InClass", cascade="all,delete")
-
     @property
     def data(self):
         return {
@@ -68,8 +67,6 @@ class Course(db.Model):
     section = db.Column(db.String(256), nullable=True)
     professor = db.Column(db.String(256), nullable=False)
 
-    in_class = db.relationship("InClass", cascade="all,delete")
-
     @property
     def total_assignments(self):
         return self.open_assignments
@@ -92,11 +89,12 @@ def data(self):
             "professor": self.professor,
             "total_assignments": self.total_assignments,
             "open_assignment": self.open_assignments,
+            "join_code": self.id[:6],
         }
 
 
-class InClass(db.Model):
-    __tablename__ = "in_class"
+class InCourse(db.Model):
+    __tablename__ = "in_course"
 
     # Foreign Keys
     owner_id = db.Column(db.String(128), db.ForeignKey(User.id), primary_key=True)
@@ -119,14 +117,17 @@ class Assignment(db.Model):
     name = db.Column(db.String(256), nullable=False, unique=True)
     hidden = db.Column(db.Boolean, default=False)
     description = db.Column(db.Text, nullable=True)
-    github_classroom_url = db.Column(db.String(256), nullable=True)
+    github_classroom_url = db.Column(db.String(256), nullable=True, default=None)
     pipeline_image = db.Column(db.String(256), unique=True, nullable=True)
     unique_code = db.Column(
         db.String(8),
         unique=True,
         default=lambda: base64.b16encode(os.urandom(4)).decode(),
     )
     ide_enabled = db.Column(db.Boolean, default=True)
+    theia_image = db.Column(
+        db.String(128), default="registry.osiris.services/anubis/theia-xv6"
+    )
 
     # Dates
     release_date = db.Column(db.DateTime, nullable=False)
@@ -143,8 +144,11 @@ def data(self):
             "id": self.id,
             "name": self.name,
             "due_date": str(self.due_date),
+            "hidden": self.hidden,
             "course": self.course.data,
             "description": self.description,
+            "ide_enabled": self.ide_enabled,
+            "ide_active": self.due_date + timedelta(days=3 * 7) > datetime.now(),
             "github_classroom_link": self.github_classroom_url,
             "tests": [t.data for t in self.tests],
         }
@@ -154,7 +158,7 @@ def meta_shape(self):
         return {
             "assignment": {
                 "name": str,
-                "class": str,
+                "course": str,
                 "unique_code": str,
                 "hidden": bool,
                 "github_classroom_url": str,
@@ -298,7 +302,16 @@ def data(self):
         """
 
         return {
+            "id": self.id,
+            "response": self.response,
             "question": self.question.data,
+        }
+
+    @property
+    def full_data(self):
+        return {
+            "id": self.id,
+            "question": self.question.full_data,
             "response": self.response,
         }
 
@@ -413,7 +426,7 @@ def data(self):
             "id": self.id,
             "assignment_name": self.assignment.name,
             "assignment_due": str(self.assignment.due_date),
-            "class_code": self.assignment.course.course_code,
+            "course_code": self.assignment.course.course_code,
             "commit": self.commit,
             "processed": self.processed,
             "state": self.state,
@@ -497,7 +510,7 @@ class SubmissionBuild(db.Model):
 
     # Fields
     stdout = db.Column(db.Text)
-    passed = db.Column(db.Boolean)
+    passed = db.Column(db.Boolean, default=None)
 
     # Timestamps
     created = db.Column(db.DateTime, default=datetime.now)
@@ -524,21 +537,25 @@ class TheiaSession(db.Model):
     __tablename__ = "theia_session"
 
     # id
-    id = default_id()
+    id = default_id(32)
 
     # Foreign keys
     owner_id = db.Column(db.String(128), db.ForeignKey(User.id), nullable=False)
     assignment_id = db.Column(
-        db.String(128), db.ForeignKey(Assignment.id), nullable=False
-    )
-    repo_id = db.Column(
-        db.String(128), db.ForeignKey(AssignmentRepo.id), nullable=False
+        db.String(128), db.ForeignKey(Assignment.id), nullable=True
     )
+    repo_url = db.Column(db.String(128), nullable=False)
 
     # Fields
     active = db.Column(db.Boolean, default=True)
     state = db.Column(db.String(128))
     cluster_address = db.Column(db.String(256), nullable=True, default=None)
+    image = db.Column(
+        db.String(128), default="registry.osiris.services/anubis/theia-xv6"
+    )
+    options = db.Column(MutableJson, nullable=False, default=lambda: dict())
+    network_locked = db.Column(db.Boolean, default=True)
+    privileged = db.Column(db.Boolean, default=False)
 
     # Timestamps
     created = db.Column(db.DateTime, default=datetime.now)
@@ -547,7 +564,6 @@ class TheiaSession(db.Model):
     last_proxy = db.Column(db.DateTime, default=datetime.now)
     last_updated = db.Column(db.DateTime, default=datetime.now, onupdate=datetime.now)
 
-    repo = db.relationship(AssignmentRepo)
     owner = db.relationship(User)
     assignment = db.relationship(Assignment)
 
@@ -558,10 +574,14 @@ def data(self):
         return {
             "id": self.id,
             "assignment_id": self.assignment_id,
-            "assignment_name": self.assignment.name,
-            "class_name": self.assignment.course.course_code,
-            "repo_id": self.repo_id,
-            "repo_url": self.repo.repo_url,
+            "assignment_name": self.assignment.name
+            if self.assignment_id is not None
+            else None,
+            "course_code": self.assignment.course.course_code
+            if self.assignment_id is not None
+            else None,
+            "netid": self.owner.netid,
+            "repo_url": self.repo_url,
             "redirect_url": theia_redirect_url(self.id, self.owner.netid),
             "active": self.active,
             "state": self.state,
@@ -582,7 +602,7 @@ class StaticFile(db.Model):
     filename = db.Column(db.String(256))
     path = db.Column(db.String(256))
     content_type = db.Column(db.String(128))
-    blob = db.Column(db.BLOB)
+    blob = db.Column(db.LargeBinary(length=(2**32)-1))
     hidden = db.Column(db.Boolean)
 
     # Timestamps
@@ -595,5 +615,7 @@ def data(self):
             "id": self.id,
             "content_type": self.content_type,
             "filename": self.filename,
+            "path": self.path,
             "hidden": self.hidden,
+            "uploaded": str(self.created)
         }