diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..76ed161 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,8 @@ +[run] +branch = True +source = + render +omit = + +[report] +fail_under=50 diff --git a/.gitignore b/.gitignore index 7bbc71c..87f7999 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,12 @@ ENV/ # mypy .mypy_cache/ + +# sass cache and maps +.sass-cache +*.css.map + +# Authentication files +bucket.json +compute.json +client.json diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..28db08f --- /dev/null +++ b/.travis.yml @@ -0,0 +1,27 @@ +sudo: required +services: + - docker +env: + - DOCKER_COMPOSE_VERSION=1.11.2 +language: python +python: + - '3.6' +before_install: + # See https://github.com/travis-ci/travis-ci/issues/7940 + - sudo rm -f /etc/boto.cfg +install: + - sudo docker --version + - sudo docker-compose --version + - pip install -r requirements.txt +jobs: + include: + - stage: test + script: ./render ci test_with_coverage + - script: ./render ci style + - script: ./render ci docs +notifications: + email: false + slack: + rooms: deptfunstuff:abJKvzApk5SKtcEyAgtswXAv + on_success: change + on_failure: change diff --git a/README.md b/README.md index 1d01a0d..5fc1e68 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,19 @@ -# render -Render Service for generating PDFs for CSUnplugged and CS Field Guide +# Render Service + +[![Build Status](https://travis-ci.org/uccser/render.svg?branch=master)](https://travis-ci.org/uccser/render)[![Documentation Status](https://readthedocs.org/projects/uccser-render/badge/?version=latest)](http://uccser-render.readthedocs.io/en/latest/?badge=latest) + +Render Service for generating PDFs for CS Unplugged and CS Field Guide. + +## Documentation + +Documentation for this project can be found on +[ReadTheDocs](http://uccser-render.readthedocs.io/en/latest/), +and can also +be built from the documentation source within the `docs/` directory. + +## Contributing + +We would love your help to make this guide the best it can be! +Please read our +[contribution guide](http://uccser-render.readthedocs.io/en/latest/getting_started/contributing_guide.html) +to get started. diff --git a/build-gcloud-image.sh b/build-gcloud-image.sh new file mode 100755 index 0000000..3de45b6 --- /dev/null +++ b/build-gcloud-image.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +function get_render_version(){ +render_version=`python3 -c 'from renderservice.render import __version__;print(__version__)'` +} + +host="gcr.io" +project_id="render-181102" +username="uccser" +get_render_version + +docker build renderservice -t "${host}/${project_id}/${username}/render:${render_version}" +gcloud docker -- push "${host}/${project_id}/${username}/render:${render_version}" diff --git a/dev/queue_client.py b/dev/queue_client.py new file mode 100644 index 0000000..c3b77ac --- /dev/null +++ b/dev/queue_client.py @@ -0,0 +1,172 @@ +"""Allows access to queue in order to create tasks and get results. + +You will need to install from the renderservice + pip install -r requirements.txt +""" + +import base64 +import importlib.machinery +import sys +import optparse +import os + + +# Local Queue Code +def get_queue_host(): + """Get the IP Address of the queue host. + + Using the command: + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' render_queue_1 + + Returns: + A string of the IP address + """ + import subprocess + command = [ + 'docker', + 'inspect', + '-f', + '\'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\'', + 'render_queue_1' + ] + result = subprocess.run(command, stdout=subprocess.PIPE) + return result.stdout.decode().strip('\n\'') + + +PROJECT_NAME = "cs-unplugged-develop" +QUEUE_NAME = "render-queue" + +QUEUE_HOST = get_queue_host() +DISCOVERY_URL = "http://{}:5052/api/{{api}}/{{apiVersion}}".format(QUEUE_HOST) +CREDENTIALS = None + + +# Online Queue Code +def get_credentials(): + """Get credentials to log into online taskqueue.""" + from oauth2client.service_account import ServiceAccountCredentials + client_secret_file = 'client.json' + scopes = ['https://www.googleapis.com/auth/cloud-platform'] + credentials = ServiceAccountCredentials.from_json_keyfile_name(client_secret_file, scopes) + return credentials + + +PROJECT_NAME = "render-181102" +LOCATION = "us-central1" +QUEUE_NAME = "render" +DISCOVERY_URL = "https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2" +CREDENTIALS = get_credentials() + + +def parse_args(): + """Command-line option parser for program control.""" + opts = optparse.OptionParser( + usage="{0} [options] [command]".format(sys.argv[0]), + description="Access and manipulate queue, where the" + "commands can be add, list, lease, document, flush.") + # Configure options + # opts.add_option("--key-file", "-k", action="store", + # type="string", help="Location of the credentials file.", + # default=None) + options, arguments = opts.parse_args() + # Extra option parsing + + # Return options + return options, arguments + + +def action_add(queue): + """Add a render task to the given queue. + + Please modify this as needed. + + Args: + queue: QueueHandler to interact with. + """ + queue.create_task({ + "kind": "task#render", + "resource_slug": "binary-cards", + "resource_name": "Binary Cards", + "resource_view": "binary_cards", + "url": "resources/binary-cards.html", + "display_numbers": False, + "black_back": True, + "paper_size": "a4", + "header_text": "", + "copies": 1, + }, tag="task") + + +def action_list(queue): + """List up to 100 tasks in the queue. + + Args: + queue: QueueHandler to interact with. + """ + tasks = list(queue.tasks()) + print("Number of tasks: {}".format(len(tasks))) + for task in tasks: + print(task) + + +def action_lease(queue): + """Lease tasks tagged with task from the queue for 30 seconds. + + Args: + queue: QueueHandler to interact with. + """ + tasks = list(queue.lease_tasks(tasks_to_fetch=100, lease_secs=30, tag="task")) + print("Number of tasks: {}".format(len(tasks))) + for task in tasks: + print(task) + + +def action_document(queue): + """Save out document results from the queue. + + Args: + queue: QueueHandler to interact with. + """ + tasks = queue.list_tasks() + for task in tasks: + if task["payload"]["kind"] == "result#document": + data = task["payload"]["document"].encode("ascii") + document = base64.b64decode(data) + filename = task["payload"]["filename"] + with open(filename, "wb") as f: + f.write(document) + + +def action_flush(queue): + """Attempt to delete all tasks from the queue. + + Args: + queue: QueueHandler to interact with. + """ + tasks = queue.tasks() + for task in tasks: + queue.delete_task(task["name"]) + + +if __name__ == "__main__": + options, arguments = parse_args() + + directory = os.path.abspath(os.path.join(os.getcwd(), os.pardir, 'renderservice/render/daemon/QueueHandler.py')) + loader = importlib.machinery.SourceFileLoader('render.daemon', directory) + handle = loader.load_module('render.daemon') + + queue = handle.QueueHandler(project_id=PROJECT_NAME, location_id=LOCATION, + taskqueue_id=QUEUE_NAME, discovery_url=DISCOVERY_URL, + credentials=CREDENTIALS) + + action = arguments[0] + if action == "add": + action_add(queue) + elif action == "list": + action_list(queue) + elif action == "lease": + action_lease(queue) + elif action == "document": + action_document(queue) + elif action == "flush": + action_flush(queue) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..eaf8f1e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,40 @@ +version: "3.2" +services: + + redis: + restart: always + image: redis + ports: + - "6379:6379" + + render: + build: + context: . + dockerfile: ./renderservice/Dockerfile-local + args: + PROCESS_PORT: 5051 + FLASK_PRODUCTION: 0 + PROJECT_NAME: cs-unplugged-develop + QUEUE_NAME: render-queue + API_DISCOVERY_URL: http://queue:5052/api/{api}/{apiVersion} + STATIC_DIRECTORY: /renderservice/static_mnt + volumes: + - ./static:/renderservice/static_mnt + ports: + - "5051:5051" + depends_on: + - queue + + queue: + build: + context: . + dockerfile: ./queueservice/Dockerfile-local + args: + PROCESS_PORT: 5052 + ports: + - "5052:5052" + environment: + - REDIS_HOST=redis + - REDIS_PORT=6379 + depends_on: + - redis diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..b94daab --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = CSUnplugged +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..e312960 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build +set SPHINXPROJ=CSUnplugged + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..50a7dab --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,10 @@ +# Check Python style +flake8==3.3.0 +pydocstyle==2.0.0 + +#Coverage Tools +coverage==4.4.1 + +# Documentation +sphinx==1.6.2 +sphinx-rtd-theme==0.2.4 diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst new file mode 100644 index 0000000..daf5bff --- /dev/null +++ b/docs/source/changelog.rst @@ -0,0 +1,41 @@ +Changelog +############################################################################## + +This page lists updates to the render service. +All notable changes to this project will be documented in this file. + +.. note :: + + We base our numbering system from the guidelines at `Semantic Versioning 2.0.0`_. + + Given a version number MAJOR.MINOR.HOTFIX: + + - MAJOR version change when major backend or text modifications are made + (for example: new topic). + - MINOR version change when content or functionality is added or updated (for + example: new videos, new activities, large number of text (typo/grammar) fixes). + - HOTFIX version change when bug hotfixes are made (for example: fixing a typo). + - A pre-release version is denoted by appending a hyphen and the alpha label + followed by the pre-release version. + +1.0.0-alpha.1 +============================================================================== + +- **Release date:** 15th September 2017 +- **Downloads:** `Source downloads are available on GitHub`_ + +**Notable changes:** + +Features: + +- + +Fixes: + +- + +.. _Semantic Versioning 2.0.0: http://semver.org/spec/v2.0.0.html +.. _Source downloads are available on GitHub: https://github.com/uccser/render/releases +.. _Hayley van Waas: https://github.com/hayleyavw +.. _Hayden Jackson: https://github.com/ravenmaster001 +.. _Jack Morgan: https://github.com/JackMorganNZ diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..1832002 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +"""Render Service documentation build configuration file. + +This file is execfile()d with the current directory set to its +containing dir. + +Note that not all possible configuration values are present in this +autogenerated file. + +All configuration values have a default; values that are commented out +serve to show the default. + +If extensions (or modules to document with autodoc) are in another directory, +add these directories to sys.path here. If the directory is relative to the +documentation root, use os.path.abspath to make it absolute, like shown here. +""" + +import os +import sys +import sphinx_rtd_theme +sys.path.insert(0, os.path.abspath('../../')) +from renderservice.render import __version__ + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'Render Service' +copyright = '2017 University of Canterbury Computer Science Education Research Group' +author = 'University of Canterbury Computer Science Education Research Group' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = __version__ +# The full version, including alpha/beta/rc tags. +release = __version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Renderdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'Renderdoc.tex', 'Render Documentation', + 'University of Canterbury Computer Science Education Research Group', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'render', 'Render Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'render', 'Render Documentation', + author, 'University of Canterbury Computer Science Education Research Group', 'One line description of project.', + 'Miscellaneous'), +] diff --git a/docs/source/developer/index.rst b/docs/source/developer/index.rst new file mode 100644 index 0000000..e459632 --- /dev/null +++ b/docs/source/developer/index.rst @@ -0,0 +1,11 @@ +Developer Documentation +############################################################################## + +The following pages are for those wanting to develop the render service. + +.. toctree:: + :maxdepth: 1 + :caption: Contents + + render + queue diff --git a/docs/source/developer/queue.rst b/docs/source/developer/queue.rst new file mode 100644 index 0000000..2ed56c0 --- /dev/null +++ b/docs/source/developer/queue.rst @@ -0,0 +1,58 @@ +Queue Service +############################################################################## + +The queue service is an image that is run during local development only, it provides a rough implementation of the Google TaskQueue RESTful API for access by the render service and external task producers. + +This should disappear with `Task Queue v2 `_ assuming there is a good way to run it locally. + +Infrastructure +============================================================================== + +When running locally using the *docker-compose* environment the Google Task Queue component is replaced with 2 other components, a Redis instance and a Queue service. + +Queue Service +------------------------------------------------------------------------------ + +The Queue Sevice mimics the `Google Task Queue REST API `_ allowing for a local task queue to be created using the Redis instance. + +Important files: + +.. code-block:: none + + queueservice/ + ├── api_data/ + | ├── __init__.py + | ├── taskqueue_v1beta2.py + | └── taskqueue_v1beta2.api + ├── Dockerfile + ├── gunicorn.conf.py + ├── requirements.txt + ├── webserver.py + └── wsgi.py + + +- ``api_data/``: Contains pairs of API specifications and Python Implementation. + + + ``taskqueue_v1beta2.py``: The python implementation of the taskqueue api for version 1beta2. + + ``taskqueue_v1beta2.api``: Google API description of the taskqueue REST API from the Google Discovery Service. This file has been modified to remove authorization scoping. + +- ``Dockerfile``: Dockerfile for building the webservice. +- ``gunicorn.conf.py``: Gunicorn configuration. +- ``requirements.txt``: Specifies required python modules needed to run the webservice. +- ``webserver.py``: Basic discovery webservice which allows for the loading of custom REST APIs. +- ``wsgi.py``: Gunicorn + Docker entrypoint for the Queue service. + +When using the Queue service it is important to note: + + - We do not expect this component to be changed much, and it is likely to be replaced in future by `Google Cloud Tasks `_. + - It is not a one-to-one mapping of the Google Task Queue REST API as it does not include ``GET`` on a specific Task Queue. + - Google error codes are not mimicked as they are undocumented, therefore the Queue Server may have more strict requirements on requests for safety but does not return error codes in the same format as Google. + - Each API call has been tested with the minimal set of body parameters for complience, but it is also possible that some requests that work locally may not work in production. + - Complex requests should be `tested here `_. + +Redis Instance +------------------------------------------------------------------------------ + +The REDIS service is currently only used by the Queue service as a datastore for tasks and handling the queuing of tasks. For those who with no knowledge of REDIS should consider it a 'high performance, in-memory database that is a glorified dictionary' for simplicity. + +For information on working with REDIS see the `REDIS documentation `_. diff --git a/docs/source/developer/render.rst b/docs/source/developer/render.rst new file mode 100644 index 0000000..fce3392 --- /dev/null +++ b/docs/source/developer/render.rst @@ -0,0 +1,116 @@ +Render Service +############################################################################## + +The render service pipline flows as follows; It accesses an external queue to get a task, consumes the task creating a resource, saves the resource and then repeats this process. Input tasks that are consumed are tagged with :code:`task` and output is handled by tasks with the :code:`result` tag. + +Task Definitions +============================================================================== + +Tasks retrieved from the render queue must be a json dictionary. That is, using the json library python must be able to load the payload as a dictionary. This dictionary must also contain a :code:`kind` mapping which specifies what can be done with the message. + +General Tasks +------------------------------------------------------------------------------ + +These tasks must be tagged with the :code:`task` string when added to the queue. The render service only consumes tasks that are tagged with :code:`task`. + +To render a resource you must use the :code:`render` task as defined below. + +.. code-block:: none + + { + kind: "task#render" + resource_slug: string, + resource_name: string, + resource_view: string, + url: string + header_text: string, + copies: 1 + } + +Where all of the above are required for every task, and for each resource additional values may be required based on their :code:`valid_options` function. + +The :code:`resource_view` determines the resource module to generate from, the :code:`resource_slug` and :code:`resource_name` are arbitary strings, the :code:`url` is preferably the url where the resource was generated from (including query), the :code:`header_text` is a string either an empty string or arbitary, and finally the :code:`copies` determines how many to generate. + +Result Tasks +------------------------------------------------------------------------------ +These tasks must be tagged with the :code:`result` string when added to the queue. The render service produces these tasks when a generate task has been completed to instruct other services where to find the output file. + +For documents that are small enough to be placed within the queue, the following task will be defined: + +.. code-block:: none + + { + kind: "result#document" + success: boolean + filename: string + document: base64 string + } + +Where :code:`success` is a boolean determining if the associated task was completed correctly, :code:`filename` is the filename of document, :code:`document` is a base64 encoded string of the document bytes. + +Another possible result is the is a document that is saved externally and a url can be used to access it, these tasks are defined as follows: + +.. code-block:: none + + { + kind: "result#link" + success: boolean + url: string + } + +Where :code:`success` is a boolean determining if the associated task was completed correctly, and :code:`url` is the address to access the document. + +Infrastructure +============================================================================== + +The render service consists of multiple processes on a single unit, this includes multiple daemons that consume tasks from an external queue and produce files, and a webserver performs health checks that monitor and restart the render daemons. + +Important files: + +.. code-block:: none + + renderservice/ + ├── render/ + | ├── daemon/ + | ├── resources/ + | ├── tests/ + | ├── webserver/ + | └── __init__.py + ├── scripts/ + | ├── docker-entrypoint.sh + | ├── mount-bucket.sh + | ├── pip-install.sh + | └── shutdown-script.sh + ├── static/ + ├── templates/ + ├── Dockerfile + ├── Dockerfile-local + ├── requirements.txt + + +- ``render/``: The python render service package. + + + ``daemon/``: Contains python classes pertaining to the daemon for consuming tasks and producing files. + + ``resources/``: Contains source files with custom logic for generating resources (pdf files). + + ``tests/``: Tests covering all the logic in the python render service package. + + ``webserver/``: Contains the webserver logic, including logic for health checks and daemon recovery. + + ``__init__.py``: Contains the version of the render service. + +- ``scripts/``: Bash shell scripts used in the creation of the render service. + + + ``docker-entrypoint.sh``: The entrypoint for the render service, creates multiple daemons and starts up the webservice. + + ``mount-bucket.sh``: Mounts the Google Cloud bucket using `gcsfuse `_. + + ``pip-install.sh``: Installs a pip requirements file in a specific order. + + ``shutdown-script.sh``: TODO: This is still to be used. A script which is run when the machine is pre-empted. + +- ``static/``: Locally stored static files, either kept locally for speed or licence reasons (such as do not distribute). +- ``templates/``: Jinja templates for webpages and render service. +- ``Dockerfile``: Dockerfile for building the service. +- ``Dockerfile-local``: Dockerfile for building the service for local development. +- ``requirements.txt``: Specifies required python modules needed to run the webservice. + +Some important things to note when working with the render service: + +- When in local development the render service does not have a live volume of the renderservice directory, that mean any changes require a rebuild of the service to see the changes. + +- The render service has multiple directories for static files, a local copy and a mounted external copy. The static folder in the root directory of the repository is mounted as the external copy when run locally. diff --git a/docs/source/getting_started/code_of_conduct.rst b/docs/source/getting_started/code_of_conduct.rst new file mode 100644 index 0000000..bcfbb2a --- /dev/null +++ b/docs/source/getting_started/code_of_conduct.rst @@ -0,0 +1,83 @@ +Code of Conduct +############################################################################## + +Our Pledge +============================================================================== + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +Our Standards +============================================================================== + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual attention + or advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic + address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +Our Responsibilities +============================================================================== + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +Scope +============================================================================== + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +Enforcement +============================================================================== + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at +```csse-education-research@canterbury.ac.nz``. +All complaints will be reviewed and investigated and will result in a response +that is deemed necessary and appropriate to the circumstances. The project +team is obligated to maintain confidentiality with regard to the reporter of +an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +Attribution +============================================================================== + +This Code of Conduct is adapted from the `Contributor Covenant`_, +`version 1.4`_. + +.. _version 1.4: http://contributor-covenant.org/version/1/4/ +.. _Contributor Covenant: http://contributor-covenant.org/ diff --git a/docs/source/getting_started/contributing_guide.rst b/docs/source/getting_started/contributing_guide.rst new file mode 100644 index 0000000..8e61351 --- /dev/null +++ b/docs/source/getting_started/contributing_guide.rst @@ -0,0 +1,229 @@ +Contributing Guide +############################################################################## + +This page lists a set of guidelines for contributing to the project. +These are just guidelines, not rules, use your best judgment and feel +free to propose changes to this document in a pull request. + +Reporting Issues and Making Suggestions +============================================================================== + +This section guides you through submitting an issue or making a suggestion +for the render service. +Following these guidelines helps maintainers and the community understand +your findings. + +Before Submitting an Issue +------------------------------------------------------------------------------ + +- `Search the issue tracker for the issue/suggestion`_ to see if it has + already been logged. + If it has, add a comment to the existing issue (even if the issue is closed) + instead of opening a new one. + +How do I Submit a Good Issue or Suggestion? +------------------------------------------------------------------------------ + +Issues are tracked in the GitHub issue tracker (if you've never used +GitHub issues before, read this `10 minute guide to become a master`_). +When creating an issue, explain the problem and include additional details to +help maintainers understand or reproduce the problem: + +For Reporting an Issue +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- **Use a clear and descriptive title** for the issue to identify the problem. +- **Clearly and concisely describe the issue** and provide screenshots if + required. +- **Link any related existing issues**. + +If the issues is a code related issue, also include the following: + +- **Describe the exact steps which reproduce the problem** in as many details + as possible. + For example, how you were generating a resource. + When listing steps, **don't just say what you did, explain how you did it**. +- **Explain which behavior you expected to see instead and why.** +- **Describe the behavior you observed after following the steps** and point + out what exactly is the problem with that behavior. +- **Can you reliably reproduce the issue?** If not, provide details about + how often the problem happens and under which conditions it normally happens. +- **Include screenshots or animated GIFs** if it helps explain the issue you + encountered. +- **What's the name and version of the OS you're using?** +- **What's the name and version of the browser you're using?** +- **If the problem is related to performance**, please provide + specifications of your computer. + +For Making a Suggestion +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Explain the suggestion and include additional details to help maintainers +understand the changes: + +- **Use a clear and descriptive title** for the issue to identify the + suggestion. +- **Clearly and concisely describe the suggestion** and provide screenshots if + required. +- **Explain why this suggestion would be useful** to most CS Unplugged users + and isn't something that should be a implemented as a community variant of + the project. +- **Link any related existing suggestions**. + +.. note:: + + **Internal Staff Only:** Assigning Issues + + Our policy is to only assign a person to an issue when they are actively + working on the issue. + Please don't assign yourself when you *plan* to do the task (for example: + in the next few days), assign yourself when you begin work. + This allows other team members to clearly see which tasks are available + to be worked on. + +Your First Code Contribution (pull request) +============================================================================== + +Unsure where to begin contributing to render service? +You can start by looking through the `issue tracker`_. + +Pull Requests +------------------------------------------------------------------------------ + +- **Include a detailed explaination** of the proposed change, including + screenshots and animated GIFs in your pull request whenever possible. +- **Read and apply the style guides** listed below. +- Your pull request should be on a new branch from our ``develop`` branch, + **that is being requested to merge back into** ``develop``. + The naming conventions of branches should be descriptive of the new + addition/modification. + Ideally they would specify their namespace as well, for example: + + - ``resource/puzzle-town`` + - ``issue/234`` + +- Link to any relevant existing issues/suggestions. +- Add necessary documentation (if appropriate). + +We aim to keep the render service as robust as possible, so please do +your best to ensure your changes won't break anything! + +Style and Etiquette Guides +============================================================================== + +Git +------------------------------------------------------------------------------ + +- Commits should be as descriptive as possible. + Other developers (and even future you) will thank you for your forethought + and verbosity for well documented commits. + Generally: + + - Limit the first line to 72 characters or less + - Reference issues and pull requests liberally + +- We use `Vincent Driessen's Git Branching Model `_ + for managing development. + Please read this document to understand our branching methods, and how + to perform clear branches and merges. + + Specifically for our respository: + + - We create a new branch for each task of work, no matter how small it is. + - We create the branch off the ``develop`` branch. + - In general, the new branch should begin with ``issue/`` followed by + the issue number. + - When a branch is completed, a pull request is created on GitHub for + review. + - Branches are merged back into ``develop``. + +GitHub +------------------------------------------------------------------------------ + +.. note:: + + Internal Staff Only + +- Mention a user (using the ``@`` symbol) when an issue is relevant to them. +- Only assign yourself to an issue, when you are actively working on it. +- A pull request requires one review approval to be merged. +- If multiple people are tagged as reviewers, we only need one review (unless + otherwise specified). +- The creator of the pull request should assign all those suitable for review. +- The creator of the pull request is the only person who should merge the pull + request. + If you approve a pull request and it shows the big green button, please + resist clicking it! + +Project Structure +------------------------------------------------------------------------------ + +- Directories should be all lowercase with dashes for spaces. +- Directories and files should use full words when named, however JavaScript, + CSS, and image directories can be named ``js/``, ``css/``, and ``img/`` + respectively. + +Text (Markdown) +------------------------------------------------------------------------------ + +- Each sentence should be started on a newline (this greatly improves + readability when comparing two states of a document). + +Programming +------------------------------------------------------------------------------ + +Quote from Google style guides: + + Be consistent. + + If you’re editing code, take a few minutes to look at the code around you + and determine its style. + If they use spaces around all their arithmetic operators, you should too. + If their comments have little boxes of hash marks around them, make your + comments have little boxes of hash marks around them too. + + The point of having style guidelines is to have a common vocabulary of coding + so people can concentrate on what you’re saying rather than on how you’re + saying it. + We present global style rules here so people know the vocabulary, but local + style is also important. + If code you add to a file looks drastically different from the existing code + around it, it throws readers out of their rhythm when they go to read it. + Avoid this. + +We aim to abide by the following style guides: + +- **Python** - We follow `PEP8`_ except for one change of line length. + `Django recommends allowing 119 characters`_, so we use this as our line + length limit. + This style is enforced by the `flake8`_ style checker. +- **HTML** - We follow the `open source HTML style guide`_ by @mdo. +- **CSS** - We follow the `open source CSS style guide`_ by @mdo. +- **JavaScript** - We follow the `Google JavaScript style guide`_. + +Licencing +------------------------------------------------------------------------------ + +Any third-party libraries or packages used within this project should have +their listed within the ``LICENCE-THIRD-PARTY`` file, with a full copy of the +licence available within the ``third-party-licences`` directory. + +Final Comments +============================================================================== + +After reading the sections above, you should be able to answer the following +questions: + +- When do I create a issue and how do I describe it? +- When and how do I create a new Git branch to work on? +- *Internal staff only:* When do I assign myself to an issue? + +.. _Search the issue tracker for the issue/suggestion: https://github.com/uccser/render/issues?utf8=%E2%9C%93&q=is%3Aissue +.. _10 minute guide to become a master: https://guides.github.com/features/issues/ +.. _issue tracker: https://github.com/uccser/render/issues +.. _PEP8: https://www.python.org/dev/peps/pep-0008/ +.. _Django recommends allowing 119 characters: https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/coding-style/ +.. _open source HTML style guide: http://codeguide.co/#html +.. _open source CSS style guide: http://codeguide.co/#css +.. _Google JavaScript style guide: https://google.github.io/styleguide/javascriptguide.xml +.. _flake8: http://flake8.pycqa.org/en/latest/ diff --git a/docs/source/getting_started/index.rst b/docs/source/getting_started/index.rst new file mode 100644 index 0000000..d0bcd1d --- /dev/null +++ b/docs/source/getting_started/index.rst @@ -0,0 +1,25 @@ +Getting Started +############################################################################## + +This documentation will help you understand how the project is setup, +basic steps on how to use it, and our guidelines for your contributions. + +.. note:: + + This project adheres to the Contributor Covenant code of conduct. + By participating, you are expected to uphold this code. + Please read our :doc:`Code of Conduct ` before continuing. + You can report unacceptable behaviour by + `emailing us `_. + +.. toctree:: + :hidden: + + code_of_conduct + +.. toctree:: + :maxdepth: 1 + :caption: Contents + + contributing_guide + installation diff --git a/docs/source/getting_started/installation.rst b/docs/source/getting_started/installation.rst new file mode 100644 index 0000000..a9ec8fc --- /dev/null +++ b/docs/source/getting_started/installation.rst @@ -0,0 +1,253 @@ +Installation Guide +################################################# + +This page will set your machine up for working on the CS Unplugged project. +You should only need to do these installation steps once (unless the required +steps for setup change). + +Requirements +================================================= + +- At least 5 GB of hard drive space. +- An internet connection to download 1 to 2 GB of data. + +Recommended Reading +================================================= + +If you aren't familiar with the following systems, we recommend +reading tutorials first on how to use them: + +- Entering terminal commands for your operating system +- Git (here are two Git tutorials: `one`_ `two`_) + +Step 1: Setup Virtual Machine (optional) +================================================= + +For those working on a computer in a restricted environment (for example: +a computer managed by an education insitution), then working in a +**virtual machine** is recommended. + +.. _step-2-install-git: + +Step 2: Install Git +================================================= + +Install the version control software `Git`_ onto your computer. + +.. note:: + + If you are new to Git and not comfortable with using the terminal, + you may like to use a free program like `SourceTree`_ to use Git. + +Step 3: Create GitHub Account +================================================= + +If you don't already have an account on GitHub, create a free account on +the `GitHub website`_. +This account will be tied to any changes you submit to the project. + +Step 4: Set Git Account Values +================================================= + +When you make a commit in Git (the term for changes to the project), the +commit is tied to a name and email address. We need to set name and email +address within the Git system installed on the machine. + +- `Setting your username in Git`_ +- `Setting your email in Git`_ + +You can also `keep your email address private on GitHub`_ if needed. + +.. note:: + + If your GitHub account is secured with two-factor authentication (2FA) + this is a perfect time to setup `SSH keys`_. + +Step 5: Download the render service repository +================================================= + +Firstly create the directory you wish to hold the render service repository +directory in if you wish to store the data in a specific location. +Once you have decided upon the location, clone (the Git term for download) the +project onto your computer. + +If you are using terminal commands to use Git, type the following command in +terminal (you don't need to enter the ``$`` character, this shows the start of +your terminal prompt): + +.. code-block:: bash + + $ git clone https://github.com/uccser/render.git + +.. note:: + + If you connect to GitHub through SSH, then type: + + .. code-block:: bash + + $ git clone git@github.com:uccser/render.git + +Once Git has cloned the directory, checkout the repository to the development +branch ``develop``. + +Step 6: Install Docker +================================================= + +We use a system called `Docker`_ to run the render service, both on local +machine for development, and also when deployed to production. +Download the latest version of the free Docker Community Edition for your +operating system from the `Docker Store`_. + +Once you have installed the software, run the following commands in a terminal +to check Docker is working as intended (you don't need to enter the ``$`` +character, this shows the start of your terminal prompt). + +.. code-block:: bash + + $ docker version + $ docker-compose version + $ docker run hello-world + +.. note:: + + Depending on your operating system, if the above commands don't work you + may need to set Docker to be able to run without ``sudo``. + You will need to do this in order to use the ``csu`` helper script. + +Step 7: Install Text Editor/IDE (optional) +================================================= + +This is a good time to install your preferred IDE or text editor, if you don't +have one already. +Some free options we love: + +- `Atom`_ +- `Sublime Text`_ + +Step 8: Install Developer Tools (optional) +================================================= + +.. note:: + + You can skip this step if you're only adding content to the project. + +For those developing the render service, you will need to install some +tools on your computer for local development. +These tools include packages for style checking and compiling documentation. + +Install Python 3 +------------------------------------------------------------------------------ + +Install Python 3 with the following command in terminal: + +.. code-block:: bash + + $ sudo apt install python3 + +Install Python 3 PIP +------------------------------------------------------------------------------ + +Then install Python 3 pip (pip is a package management system used to +install and manage software packages written in Python) with the following +command in terminal: + +.. code-block:: bash + + $ sudo apt install python3-pip + +Install Python virtualenv +------------------------------------------------------------------------------ + +We recommend (though it's not required) to work within a virtual environment. +This helps to prevent conflicts with dependencies. + +Install virtualenv with the following command in terminal: + +.. code-block:: bash + + $ sudo pip3 install virtualenv + +.. note:: + + **Optional step:** You can also install `virtualenvwrapper`_ to make it + easier when using and managing your virtual environments. + +Create Virtual Environment +------------------------------------------------------------------------------ + +Type the following commands in terminal to create and activate +a virtualenv named ``venv``. +You can change the virtual environment name to whatever you wish. +You will need to replace the ``x`` with the version number of Python you +have (for example: ``python3.5``): + +.. code-block:: bash + + $ python -m virtualenv --python=python3.x venv + $ . venv/bin/activate + +.. note:: + + If you installed ``virtualenvwrapper``, then type the following command to + to create a virtual environment called ``csunplugged``, with Python within + the virtual environment already set to Python 3. + + .. code-block:: bash + + $ mkvirtualenv --python=/usr/bin/python3.x csunplugged + +You should now have the name of your virtual environment before the terminal +prompt. + +Install Packages into the Virtual Environemnt +------------------------------------------------------------------------------ + +Now that the virtual environment is active, we can install the Python packages +into it for local development. +This allows you to run these tools without having to run these within the +Docker system. + +.. code-block:: bash + + $ pip install -r requirements/local.txt + +.. _installation-check-project-setup-works: + +Step 9: Check Project Setup Works +================================================= + +To check the project works, open a terminal in the project root directory, +which is the ``cs-unplugged/`` directory (should contain a file called +``csu``). + +Type the following command into the terminal (we will cover this command +in more detail on the next page): + +.. code-block:: bash + + $ ./render start + +If this is the first time you're running this script, it will need to build +system images. + +After the helper script builds the system images, it will automatically start +the system, and will let you know when the system is ready. + +Congratulations if you made it this far and everything is working, +you're all set to contribute to the render service. + +.. _one: https://git-scm.com/docs/gittutorial +.. _two: https://try.github.io/levels/1/challenges/1 +.. _virtualenvwrapper: https://virtualenvwrapper.readthedocs.io/en/latest/ +.. _Git: https://git-scm.com/ +.. _SourceTree: https://www.sourcetreeapp.com/ +.. _GitHub website: https://github.com/ +.. _SSH keys: https://help.github.com/articles/connecting-to-github-with-ssh/ +.. _Setting your username in Git: https://help.github.com/articles/setting-your-username-in-git/ +.. _Setting your email in Git: https://help.github.com/articles/setting-your-email-in-git/ +.. _keep your email address private on GitHub: https://help.github.com/articles/keeping-your-email-address-private/ +.. _Docker: https://www.docker.com/ +.. _Docker Store: https://store.docker.com/search?type=edition&offering=community +.. _Verto documentation: http://verto.readthedocs.io/en/latest/install.html +.. _Atom: https://atom.io/ +.. _Sublime Text: https://www.sublimetext.com/ diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..33e143e --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,40 @@ +Welcome to Render +############################################################################## + +Welcome to the documentation for the Render project for CS Unplugged and CS Field Guide. The documentation is split into the following sections: + +------------------------------------------------------------------------------ + +:doc:`Getting Started Documentation ` +============================================================================== + +This documentation contains: + +- Our expectations of your contributions to the project +- Steps to install the project on your system +- Explainations on how the project is setup +- Details on basic commands to use the system + +------------------------------------------------------------------------------ + +:doc:`Developer Documentation ` +============================================================================== + +This documentation is for those who want to contribute to developing the +render service for generating PDF resources. +This is the documentation to read if you want to do any of the +following (or similar): + +- Add a generatable PDF resource +- Add to the RESTful APIs +- Contribute to test suite + +------------------------------------------------------------------------------ + +.. toctree:: + :maxdepth: 4 + :caption: Table of Contents + + getting_started/index + developer/index + changelog diff --git a/queue.yaml b/queue.yaml new file mode 100644 index 0000000..4d5313b --- /dev/null +++ b/queue.yaml @@ -0,0 +1,3 @@ +queue: +- name: render + mode: pull diff --git a/queueservice/Dockerfile-local b/queueservice/Dockerfile-local new file mode 100644 index 0000000..804ff74 --- /dev/null +++ b/queueservice/Dockerfile-local @@ -0,0 +1,20 @@ +FROM uccser/python:3.6.2-stretch +LABEL maintainer="csse-education-research@canterbury.ac.nz" + +# Arguments and Environment Variables +ARG DEBIAN_FRONTEND=noninteractive +ARG PROCESS_PORT=5052 +ENV PORT ${PROCESS_PORT} + +# Install dependencies +COPY queueservice/requirements.txt / +RUN /docker_venv/bin/pip3 install -r /requirements.txt + +# Setup Working Directory +RUN mkdir /queue +ADD ./queueservice /queue +WORKDIR /queue + +EXPOSE ${PORT} + +ENTRYPOINT ["/queue/docker-entrypoint.sh"] diff --git a/queueservice/api_data/__init__.py b/queueservice/api_data/__init__.py new file mode 100644 index 0000000..f5462e7 --- /dev/null +++ b/queueservice/api_data/__init__.py @@ -0,0 +1 @@ +"""API Module.""" diff --git a/queueservice/api_data/taskqueue_v1beta2.api b/queueservice/api_data/taskqueue_v1beta2.api new file mode 100644 index 0000000..689f8d7 --- /dev/null +++ b/queueservice/api_data/taskqueue_v1beta2.api @@ -0,0 +1,528 @@ +{ + "kind": "discovery#restDescription", + "etag": "\"8c21c3201a56f5974ecbe3140c7b8996\"", + "discoveryVersion": "v1", + "id": "taskqueue:v1beta2", + "name": "taskqueue", + "version": "v1beta2", + "revision": "20160428", + "title": "TaskQueue API", + "description": "Accesses a Google App Engine Pull Task Queue over REST.", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "https://www.google.com/images/icons/product/app_engine-16.png", + "x32": "https://www.google.com/images/icons/product/app_engine-32.png" + }, + "documentationLink": "https://developers.google.com/appengine/docs/python/taskqueue/rest", + "protocol": "rest", + "baseUrl": "https://www.googleapis.com/taskqueue/v1beta2/projects/", + "basePath": "/taskqueue/v1beta2/projects/", + "rootUrl": "https://www.googleapis.com/", + "servicePath": "taskqueue/v1beta2/projects/", + "batchPath": "batch", + "parameters": { + "alt": { + "type": "string", + "description": "Data format for the response.", + "default": "json", + "enum": [ + "json" + ], + "enumDescriptions": [ + "Responses with Content-Type of application/json" + ], + "location": "query" + }, + "fields": { + "type": "string", + "description": "Selector specifying which fields to include in a partial response.", + "location": "query" + }, + "key": { + "type": "string", + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", + "location": "query" + }, + "oauth_token": { + "type": "string", + "description": "OAuth 2.0 token for the current user.", + "location": "query" + }, + "prettyPrint": { + "type": "boolean", + "description": "Returns response with indentations and line breaks.", + "default": "true", + "location": "query" + }, + "quotaUser": { + "type": "string", + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. Overrides userIp if both are provided.", + "location": "query" + }, + "userIp": { + "type": "string", + "description": "IP address of the site where the request originates. Use this if you want to enforce per-user limits.", + "location": "query" + } + }, + "auth": { + "oauth2": {} + }, + "schemas": { + "Task": { + "id": "Task", + "type": "object", + "properties": { + "enqueueTimestamp": { + "type": "string", + "description": "Time (in seconds since the epoch) at which the task was enqueued.", + "format": "int64" + }, + "id": { + "type": "string", + "description": "Name of the task." + }, + "kind": { + "type": "string", + "description": "The kind of object returned, in this case set to task.", + "default": "taskqueues#task" + }, + "leaseTimestamp": { + "type": "string", + "description": "Time (in seconds since the epoch) at which the task lease will expire. This value is 0 if the task isnt currently leased out to a worker.", + "format": "int64" + }, + "payloadBase64": { + "type": "string", + "description": "A bag of bytes which is the task payload. The payload on the JSON side is always Base64 encoded." + }, + "queueName": { + "type": "string", + "description": "Name of the queue that the task is in." + }, + "retry_count": { + "type": "integer", + "description": "The number of leases applied to this task.", + "format": "int32" + }, + "tag": { + "type": "string", + "description": "Tag for the task, could be used later to lease tasks grouped by a specific tag." + } + } + }, + "TaskQueue": { + "id": "TaskQueue", + "type": "object", + "properties": { + "acl": { + "type": "object", + "description": "ACLs that are applicable to this TaskQueue object.", + "properties": { + "adminEmails": { + "type": "array", + "description": "Email addresses of users who are \"admins\" of the TaskQueue. This means they can control the queue, eg set ACLs for the queue.", + "items": { + "type": "string" + } + }, + "consumerEmails": { + "type": "array", + "description": "Email addresses of users who can \"consume\" tasks from the TaskQueue. This means they can Dequeue and Delete tasks from the queue.", + "items": { + "type": "string" + } + }, + "producerEmails": { + "type": "array", + "description": "Email addresses of users who can \"produce\" tasks into the TaskQueue. This means they can Insert tasks into the queue.", + "items": { + "type": "string" + } + } + } + }, + "id": { + "type": "string", + "description": "Name of the taskqueue." + }, + "kind": { + "type": "string", + "description": "The kind of REST object returned, in this case taskqueue.", + "default": "taskqueues#taskqueue" + }, + "maxLeases": { + "type": "integer", + "description": "The number of times we should lease out tasks before giving up on them. If unset we lease them out forever until a worker deletes the task.", + "format": "int32" + }, + "stats": { + "type": "object", + "description": "Statistics for the TaskQueue object in question.", + "properties": { + "leasedLastHour": { + "type": "string", + "description": "Number of tasks leased in the last hour.", + "format": "int64" + }, + "leasedLastMinute": { + "type": "string", + "description": "Number of tasks leased in the last minute.", + "format": "int64" + }, + "oldestTask": { + "type": "string", + "description": "The timestamp (in seconds since the epoch) of the oldest unfinished task.", + "format": "int64" + }, + "totalTasks": { + "type": "integer", + "description": "Number of tasks in the queue.", + "format": "int32" + } + } + } + } + }, + "Tasks": { + "id": "Tasks", + "type": "object", + "properties": { + "items": { + "type": "array", + "description": "The actual list of tasks returned as a result of the lease operation.", + "items": { + "$ref": "Task" + } + }, + "kind": { + "type": "string", + "description": "The kind of object returned, a list of tasks.", + "default": "taskqueue#tasks" + } + } + }, + "Tasks2": { + "id": "Tasks2", + "type": "object", + "properties": { + "items": { + "type": "array", + "description": "The actual list of tasks currently active in the TaskQueue.", + "items": { + "$ref": "Task" + } + }, + "kind": { + "type": "string", + "description": "The kind of object returned, a list of tasks.", + "default": "taskqueues#tasks" + } + } + } + }, + "resources": { + "taskqueues": { + "methods": { + "get": { + "id": "taskqueue.taskqueues.get", + "path": "{project}/taskqueues/{taskqueue}", + "httpMethod": "GET", + "description": "Get detailed information about a TaskQueue.", + "parameters": { + "getStats": { + "type": "boolean", + "description": "Whether to get stats. Optional.", + "location": "query" + }, + "project": { + "type": "string", + "description": "The project under which the queue lies.", + "required": true, + "location": "path" + }, + "taskqueue": { + "type": "string", + "description": "The id of the taskqueue to get the properties of.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "taskqueue" + ], + "response": { + "$ref": "TaskQueue" + } + } + } + }, + "tasks": { + "methods": { + "delete": { + "id": "taskqueue.tasks.delete", + "path": "{project}/taskqueues/{taskqueue}/tasks/{task}", + "httpMethod": "DELETE", + "description": "Delete a task from a TaskQueue.", + "parameters": { + "project": { + "type": "string", + "description": "The project under which the queue lies.", + "required": true, + "location": "path" + }, + "task": { + "type": "string", + "description": "The id of the task to delete.", + "required": true, + "location": "path" + }, + "taskqueue": { + "type": "string", + "description": "The taskqueue to delete a task from.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "taskqueue", + "task" + ] + }, + "get": { + "id": "taskqueue.tasks.get", + "path": "{project}/taskqueues/{taskqueue}/tasks/{task}", + "httpMethod": "GET", + "description": "Get a particular task from a TaskQueue.", + "parameters": { + "project": { + "type": "string", + "description": "The project under which the queue lies.", + "required": true, + "location": "path" + }, + "task": { + "type": "string", + "description": "The task to get properties of.", + "required": true, + "location": "path" + }, + "taskqueue": { + "type": "string", + "description": "The taskqueue in which the task belongs.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "taskqueue", + "task" + ], + "response": { + "$ref": "Task" + } + }, + "insert": { + "id": "taskqueue.tasks.insert", + "path": "{project}/taskqueues/{taskqueue}/tasks", + "httpMethod": "POST", + "description": "Insert a new task in a TaskQueue", + "parameters": { + "project": { + "type": "string", + "description": "The project under which the queue lies", + "required": true, + "location": "path" + }, + "taskqueue": { + "type": "string", + "description": "The taskqueue to insert the task into", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "taskqueue" + ], + "request": { + "$ref": "Task" + }, + "response": { + "$ref": "Task" + } + }, + "lease": { + "id": "taskqueue.tasks.lease", + "path": "{project}/taskqueues/{taskqueue}/tasks/lease", + "httpMethod": "POST", + "description": "Lease 1 or more tasks from a TaskQueue.", + "parameters": { + "groupByTag": { + "type": "boolean", + "description": "When true, all returned tasks will have the same tag", + "location": "query" + }, + "leaseSecs": { + "type": "integer", + "description": "The lease in seconds.", + "required": true, + "format": "int32", + "location": "query" + }, + "numTasks": { + "type": "integer", + "description": "The number of tasks to lease.", + "required": true, + "format": "int32", + "location": "query" + }, + "project": { + "type": "string", + "description": "The project under which the queue lies.", + "required": true, + "location": "path" + }, + "tag": { + "type": "string", + "description": "The tag allowed for tasks in the response. Must only be specified if group_by_tag is true. If group_by_tag is true and tag is not specified the tag will be that of the oldest task by eta, i.e. the first available tag", + "location": "query" + }, + "taskqueue": { + "type": "string", + "description": "The taskqueue to lease a task from.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "taskqueue", + "numTasks", + "leaseSecs" + ], + "response": { + "$ref": "Tasks" + } + }, + "list": { + "id": "taskqueue.tasks.list", + "path": "{project}/taskqueues/{taskqueue}/tasks", + "httpMethod": "GET", + "description": "List Tasks in a TaskQueue", + "parameters": { + "project": { + "type": "string", + "description": "The project under which the queue lies.", + "required": true, + "location": "path" + }, + "taskqueue": { + "type": "string", + "description": "The id of the taskqueue to list tasks from.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "taskqueue" + ], + "response": { + "$ref": "Tasks2" + } + }, + "patch": { + "id": "taskqueue.tasks.patch", + "path": "{project}/taskqueues/{taskqueue}/tasks/{task}", + "httpMethod": "PATCH", + "description": "Update tasks that are leased out of a TaskQueue. This method supports patch semantics.", + "parameters": { + "newLeaseSeconds": { + "type": "integer", + "description": "The new lease in seconds.", + "required": true, + "format": "int32", + "location": "query" + }, + "project": { + "type": "string", + "description": "The project under which the queue lies.", + "required": true, + "location": "path" + }, + "task": { + "type": "string", + "required": true, + "location": "path" + }, + "taskqueue": { + "type": "string", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "taskqueue", + "task", + "newLeaseSeconds" + ], + "request": { + "$ref": "Task" + }, + "response": { + "$ref": "Task" + } + }, + "update": { + "id": "taskqueue.tasks.update", + "path": "{project}/taskqueues/{taskqueue}/tasks/{task}", + "httpMethod": "POST", + "description": "Update tasks that are leased out of a TaskQueue.", + "parameters": { + "newLeaseSeconds": { + "type": "integer", + "description": "The new lease in seconds.", + "required": true, + "format": "int32", + "location": "query" + }, + "project": { + "type": "string", + "description": "The project under which the queue lies.", + "required": true, + "location": "path" + }, + "task": { + "type": "string", + "required": true, + "location": "path" + }, + "taskqueue": { + "type": "string", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "taskqueue", + "task", + "newLeaseSeconds" + ], + "request": { + "$ref": "Task" + }, + "response": { + "$ref": "Task" + } + } + } + } + } +} diff --git a/queueservice/api_data/taskqueue_v1beta2.py b/queueservice/api_data/taskqueue_v1beta2.py new file mode 100644 index 0000000..1280726 --- /dev/null +++ b/queueservice/api_data/taskqueue_v1beta2.py @@ -0,0 +1,703 @@ +"""Mimicked API for the Google Taskqueue API v1beta2.""" +import os +import time +import json +import uuid +import redis +from flask import Blueprint, request + +# REDIS SETUP +redis_host = os.getenv("REDIS_HOST", "localhost") +redis_port = int(os.getenv("REDIS_PORT", 6379)) +redis_pool = redis.ConnectionPool(host=redis_host, port=redis_port, db=0) +r = redis.StrictRedis(connection_pool=redis_pool) + + +# Models +def generate_id(): + """Get a unique identifier. + + Returns: + A string of an uuid that does not exist in the redis server. + """ + id = str(uuid.uuid4()) + while r.exists(id): + id = str(uuid.uuid4()) + return id + + +def now(): + """Get the current time since the epoch. + + Returns: + An integer of the current time since the epoch in microseconds. + """ + return int(time.time() * 10**6) + + +class Task(object): + """Describes a Google Task. + + With methods for creation from json with error check and automatic + property building on creation. + """ + + def __init__(self, queueName, payloadBase64, id=None, kind="taskqueues#task", + enqueueTimestamp=None, leaseTimestamp=None, retry_count=0, tag=""): + """Create a new Task object. + + Where only the queueName and payloadBase64 are required to + make a new object. + + Args: + queueName: A string of the name of the associated queue. (str) + payloadBase64: A string of the user-defined payload as a + string in base64. (base64 string) + id: A string of the task id, if None is automatically + generated. (str) + kind: A string of the kind of object. (str) + enqueueTimestamp: A long integer of the timesince the epoch the + task was enqueued. (long) + leaseTimestamp: A long integer of the time the task becomes + avaliable for leasing. (long) + retry_count: The number of times a task was leased. (int) + tag: The tag associated with the tag. (str) + """ + self.queueName = queueName + self.payloadBase64 = payloadBase64 + + self.id = id if id is not None else generate_id() + self.enqueueTimestamp = enqueueTimestamp if enqueueTimestamp is not None else now() + self.leaseTimestamp = leaseTimestamp if leaseTimestamp is not None else now() + self.kind = kind + self.retry_count = retry_count + self.tag = tag + + def _asdict(self): + """Convert the current object into a dictionary. + + Returns: + A dictionary of properties mapped to keys. + """ + return { + "id": self.id, + "kind": self.kind, + "enqueueTimestamp": self.enqueueTimestamp, + "leaseTimestamp": self.leaseTimestamp, + "payloadBase64": self.payloadBase64, + "queueName": self.queueName, + "retry_count": self.retry_count, + "tag": self.tag + } + + def to_json(self): + """Convert the current object into a json dictionary. + + Returns: + A dictionary where the keys match the properties of the + object. + """ + return self._asdict() + + @staticmethod + def from_json(json): + """Create a new Task object from a json dictionary. + + Includes error checking that all the required properties + are present. + + Args: + json: A dictionary where keys map to Task object + properties. (dict) + Returns: + The converted Task object. + """ + keys = [ + "id", "kind", "enqueueTimestamp", "leaseTimestamp", "payloadBase64", + "queueName", "retry_count", "tag" + ] + if any(key not in json for key in keys): + raise Exception("Not all values specified.") + return Task(**json) + + +class Stats(object): + """Describes a Stats object used in the Google TaskQueue. + + The methods for creation from json with error check and automatic + property building on creation. + + TODO: We currently do not update our stats objects. + """ + + def __init__(self, leasedLastHour=0, leasedLastMinute=0, oldestTask=0, totalTasks=0): + """Create a new Stats object. + + Args: + leasedLastHour: The number of tasks leased in the last hour. (int) + leasedLastMinute: The number of tasks leased in the last minute. (int) + oldestTask: The age of the oldest task. (float) + totalTasks: The number of the tasks in the queue. (int) + """ + self.leasedLastHour = leasedLastHour + self.leasedLastMinute = leasedLastMinute + self.oldestTask = oldestTask + self.totalTasks = totalTasks + + def _asdict(self): + """Convert the current object into a dictionary. + + Returns: + A dictionary of properties mapped to keys. (dict) + """ + return { + "leasedLastHour": self.leasedLastHour, + "leasedLastMinute": self.leasedLastMinute, + "oldestTask": self.oldestTask, + "totalTasks": self.totalTasks + } + + def to_json(self): + """Convert the current object into a json dictionary. + + Returns: + A dictionary where the keys match the properties of the + object. (dict) + """ + return self._asdict() + + @staticmethod + def from_json(json): + """Create a new Stats object from a json dictionary. + + Includes error checking that all the required properties + are present. + + Args: + json: A dictionary where keys map to Task object + properties. (dict) + Returns: + The converted Stats object. (class Stats) + """ + keys = [ + "leasedLastHour", "leasedLastMinute", "oldestTask", "totalTasks" + ] + if any(key not in json for key in keys): + raise Exception("Not all values specified.") + return Stats(**json) + + +class Acl(object): + """Describes the user-rights over the associated object. + + TODO: We currently do not explicitly use the Acl object. + """ + + def __init__(self, adminEmails=None, consumerEmails=None, producerEmails=None): + """Create a new Acl object. + + Args: + adminEmails: A list of administrator emails (list of strings). + consumerEmails: A list of emails with access to read/delete (list of strings). + producerEmails: A list of emails with access to write (list of strings). + """ + self.adminEmails = adminEmails if adminEmails is not None else [] + self.consumerEmails = consumerEmails if consumerEmails is not None else [] + self.producerEmails = producerEmails if producerEmails is not None else [] + + def _asdict(self): + """Convert the current object into a dictionary. + + Returns: + A dictionary of properties mapped to keys (dict). + """ + return { + "adminEmails": self.adminEmails, + "consumerEmails": self.consumerEmails, + "producerEmails": self.producerEmails + } + + def to_json(self): + """Convert the current object into a json dictionary. + + Returns: + A dictionary where the keys match the properties of the + object. (dict) + """ + return self._asdict() + + @staticmethod + def from_json(json): + """Create a new Acl object from a json dictionary. + + Includes error checking that all the required properties + are present. + + Args: + json: A dictionary where keys map to Task object + properties. (dict) + Returns: + The converted Acl object. (class ACL) + """ + keys = [ + "adminEmails", "consumerEmails", "producerEmails" + ] + if any(key not in json for key in keys): + raise Exception("Not all values specified.") + return Acl(**json) + + +class TaskQueue(object): + """Describes a Google TaskQueue. + + With methods for creation from json with error check and automatic + property building on creation. + + TODO: Current implementation does not make use of stats and maxLeases etc. + """ + + def __init__(self, id=None, kind="taskqueues#taskqueue", maxLeases=None, stats=None, acl=None): + """Create a new TaskQueue object. + + Args: + id: The identifier of the taskqueue, if None is given a + identifier is generated. (str) + kind: The kind of the taskqueue. (str) + maxLeases: The max number of leases an item can take before + it is removed from the queue. (int) + stats: An Stats object recording information on the queue. (class Stats) + acl: An Acl object of Authorised users of the queue. (class Acl) + """ + self.id = id if id is not None else generate_id() + self.kind = kind + self.maxLeases = maxLeases + self.stats = stats if stats is not None else Stats() + self.acl = acl if acl is not None else Acl() + + def _asdict(self): + """Convert the current object into a dictionary. + + Returns: + A dictionary of properties mapped to keys. (dict) + """ + return { + "id": self.id, + "kind": self.kind, + "maxLeases": self.maxLeases, + "stats": self.stats, + "acl": self.acl + } + + def to_json(self): + """Convert the current object into a json dictionary. + + Returns: + A dictionary where the keys match the properties of the + object. (dict) + """ + d = self._asdict() + d["stats"] = self.stats.to_json() + d["acl"] = self.acl.to_json() + return d + + @staticmethod + def from_json(json): + """Create a new TaskQueue object from a json dictionary. + + Includes error checking that all the required properties + are present. + + Args: + json: A dictionary where keys map to Task object + properties. (dict) + Returns: + The converted TaskQueue object. (class TaskQueue) + """ + keys = [ + "id", "kind", "maxLeases", "stats", "acl" + ] + if any(key not in json for key in keys): + raise Exception("Not all values specified.") + + q = TaskQueue(**json) + q.stats = Stats.from_json(json["stats"]) + q.acl = Acl.from_json(json["acl"]) + return q + + +# Taskqueue Blueprint +taskqueue_v1beta2_api = Blueprint("taskqueue.v1beta2", __name__) + + +@taskqueue_v1beta2_api.route( + "//taskqueues/", + methods=["GET", "POST"]) +def taskqueue_api(project=None, taskqueue=None): + """List details on the given taskqueue. + + Currently stats such as leasedLastMinute, leasedLastHour + are not tracked as they are not useful. + + Args: + project: A string of the project to work on. (str) + taskqueue: A string of the affected taskqueue. (str) + + GET: + List the details describing the task queue. + """ + if project is None: + return "You must specify a project.", 400 + elif taskqueue is None: + return "You must specify a taskqueue.", 400 + + project = project.replace("b~", "") + queue_key = ".".join([project, taskqueue]) + + if request.method == "GET": + taskqueue_id = "projects/b~{}/taskqueues/{}".format(project, taskqueue) + response = { + "kind": "taskqueues#taskqueue", + "id": taskqueue_id, + } + + getStats = request.args.get("getStats") + if getStats: + totalTasks = r.zcard(queue_key) + + keys = r.zrange(name=queue_key, start=0, end=1) + oldestTask = 0 + if len(keys) != 0: + key = keys[0] + oldestTask = r.zscore(queue_key, key) + + response["stats"] = { + "totalTasks": totalTasks, + "oldestTask": oldestTask, + "leasedLastMinute": 0, + "leasedLastHour": 0 + } + return json.dumps(response) + return "", 404 + + +@taskqueue_v1beta2_api.route( + "//taskqueues//tasks", + methods=["GET", "POST"]) +def tasks_api(project=None, taskqueue=None): + """List items or Insert into a given queue. + + Args: + project: A string of the project to work on. (str) + taskqueue: A string of the affected taskqueue. (str) + + GET: + Lists all non-deleted Tasks in a TaskQueue, whether or not + they are currently leased, up to a maximum of 100. + + Returns: + A json object containing a kind and items attribute. + Where items is a list of tasks. + + POST: + Inserts a task into an existing queue. + + Body: + Must be a JSON Task object, where only the queueName + and payloadBase64 need to be specified. + Returns: + A json object of the created Task. + """ + if project is None: + return "You must specify a project.", 400 + elif taskqueue is None: + return "You must specify a taskqueue.", 400 + + project = project.replace("b~", "") + queue_key = ".".join([project, taskqueue]) + + # Lists all non-deleted Tasks in a TaskQueue, whether or not + # they are currently leased, up to a maximum of 100. + if request.method == "GET": + start = 0 + tasks = [] + numTasks = 100 + while len(tasks) < numTasks: + tasks_needed = numTasks - len(tasks) + end = max(start, start + tasks_needed - 1) + keys = r.zrange(name=queue_key, start=start, end=end) + if len(keys) == 0: + break + + for key in keys: # Could become parallelizable + task_string = r.get(name=key) + if task_string is None: + continue + + task_string = task_string.decode() + task_json = json.loads(task_string) + tasks.append(task_json) + + start = end + 1 + + return json.dumps({ + "kind": "taskqueue#tasks", + "items": tasks + }) + + # Insert a task into an existing queue. + elif request.method == "POST": + task_json = json.loads(request.get_data().decode()) + + task = Task(**task_json) + task_string = json.dumps(task.to_json()) + task_id = task.id + + if r.exists(name=task_id): + return "Task name is invalid.", 400 + + p = r.pipeline() + p.zadd(queue_key, task.enqueueTimestamp, task_id) + p.set(name=task_id, value=task_string) + p.execute() + + return task_string + + +@taskqueue_v1beta2_api.route( + "//taskqueues//tasks/lease", + methods=["POST"]) +def lease_api(project=None, taskqueue=None): + """Lease items from the taskqueue. + + Args: + project: A string of the project to work on. (str) + taskqueue: A string of the affected taskqueue. (str) + + POST: + Acquires a lease on the topmost N unowned tasks in the + specified queue. + + Query: + leaseSecs: An integer of the number of seconds to lease + the tasks for. + numTasks: An integer of the number of tasks to lease. + groupByTag: (Optional) true or false, determining whether + to get tasks by tag. + tag: (Optional) A string specifing which tag leased tasks + must have. + Returns: + A json object containing a kind and items attribute. + Where items is a list of tasks. + """ + if project is None: + return "You must specify a project.", 400 + elif taskqueue is None: + return "You must specify a taskqueue.", 400 + + project = project.replace("b~", "") + queue_key = ".".join([project, taskqueue]) + + # Acquires a lease on the topmost N unowned tasks in the specified + # queue. + # Required query parameters: leaseSecs, numTasks + if request.method == "POST": + leaseSecs = request.args.get("leaseSecs") + numTasks = request.args.get("numTasks") + if leaseSecs is None or numTasks is None: + return "Missing leaseSecs and numTasks in query.", 400 + + groupByTag = request.args.get("groupByTag", False) + tag = request.args.get("tag", "") + if not groupByTag and tag != "": + return "In query tag specified without groupByTag.", 400 + + # load tag of the oldest task + if groupByTag and "tag" not in request.args.keys(): + keys = r.zrange(name=queue_key, start=0, end=0) + if len(keys) == 1: + key = keys[0] + task_string = r.get(name=key) + if task_string is not None: + task_string = task_string.decode() + task_json = json.loads(task_string) + task = Task.from_json(task_json) + tag = task.tag + + start = 0 + tasks = [] + now_milliseconds = now() + numTasks = min(int(numTasks), 1000) + while len(tasks) < numTasks: + tasks_needed = numTasks - len(tasks) + end = max(start, start + tasks_needed - 1) + keys = r.zrange(name=queue_key, start=start, end=end) + if len(keys) == 0: + break + + for key in keys: # Could become parallelizable + task_string = r.get(name=key) + if task_string is None: + continue + + task_string = task_string.decode() + task_json = json.loads(task_string) + task = Task.from_json(task_json) + + if groupByTag and task.tag != tag: + continue + + if task.leaseTimestamp > now_milliseconds: + continue + + task.leaseTimestamp = now_milliseconds + int(leaseSecs) * 10**6 + task_json = task.to_json() + tasks.append(task_json) + + task.retry_count += 1 + task_string = json.dumps(task.to_json()) + r.set(name=key, value=task_string) + + start = end + 1 + + return json.dumps({ + "kind": "taskqueue#tasks", + "items": tasks + }) + + +@taskqueue_v1beta2_api.route( + "//taskqueues//tasks/", + methods=["GET", "POST", "PATCH", "DELETE"]) +def task_api(project=None, taskqueue=None, task_id=None): + """Get, Update, or Delete a task. + + Args: + project: A string of the project to work on. (str) + taskqueue: A string of the affected taskqueue. (str) + task_id: The id of the task to modify. (str) + + GET: + Gets the named task in a TaskQueue. + + Returns: + A task json object. + + POST: + Update the duration of a task lease. + + Query: + newLeaseSeconds: The number of seconds to complete the + task from now. + Body: + The json object of the task to update. + Returns: + The updated json object of the task. + + PATCH: + Update the duration of a task lease. + + Query: + newLeaseSeconds: The number of seconds to complete the + task from now. + Body: + The json object of the task to update where the required + attributes are minimal and only needs queue-name. + Returns: + The updated json object of the task. + + DELETE: + Deletes a task from a TaskQueue. + + Returns: + A empty success status i.e 204. + """ + if project is None: + return "You must specify a project.", 400 + elif taskqueue is None: + return "You must specify a taskqueue.", 400 + elif task_id is None: + return "You must specify something to add.", 400 + + project = project.replace("b~", "") + queue_key = ".".join([project, taskqueue]) + + # Gets the named task in a TaskQueue. + if request.method == "GET": + task_string = r.get(name=task_id) + if task_string is None: + return "Task name does not exist.", 400 + return task_string.decode() + + # Update the duration of a task lease. + # Required query parameters: newLeaseSeconds + elif request.method == "POST": + newLeaseSeconds = request.args.get("newLeaseSeconds", None) + if newLeaseSeconds is None: + return "newLeaseSeconds is required.", 400 + if not newLeaseSeconds.isdecimal(): + return "newLeaseSeconds must be an integer.", 400 + newLeaseSeconds = int(newLeaseSeconds) + + task_string = r.get(name=task_id) + if task_string is None: + return "Task name is invalid.", 400 + + task_string = task_string.decode() + task_json = json.loads(task_string) + task = Task.from_json(task_json) + if now() > task.leaseTimestamp: + return "The task lease has expired.", 400 + + input_json = json.loads(request.get_data().decode()) + input_task = Task.from_json(input_json) + if input_task.queueName != taskqueue: + return "Cannot change task in a different queue.", 400 + if input_task.id != task.id: + return "Task IDs must match.", 400 + + newLeaseTimestamp = now() + newLeaseSeconds * 10**6 + task.leaseTimestamp = newLeaseTimestamp + task_string = json.dumps(task.to_json()) + + r.set(name=task_id, value=task_string) + return task_string + + # Update tasks that are leased out of a TaskQueue. + # Required query parameters: newLeaseSeconds + elif request.method == "PATCH": + newLeaseSeconds = request.args.get("newLeaseSeconds", None) + if newLeaseSeconds is None: + return "newLeaseSeconds is required.", 400 + if not newLeaseSeconds.isdecimal(): + return "newLeaseSeconds must be an integer.", 400 + newLeaseSeconds = int(newLeaseSeconds) + + task_string = r.get(name=task_id) + if task_string is None: + return "Task name is invalid.", 400 + + task_string = task_string.decode() + task_json = json.loads(task_string) + task = Task.from_json(task_json) + if now() > task.leaseTimestamp: + return "The task lease has expired.", 400 + + # minimal only needs queue-name + input_json = json.loads(request.get_data().decode()) + if input_json["queueName"] != taskqueue: + return "Cannot change task in a different queue.", 400 + + newLeaseTimestamp = now() + newLeaseSeconds * 10**6 + task.leaseTimestamp = newLeaseTimestamp + task_string = json.dumps(task.to_json()) + + r.set(name=task_id, value=task_string) + return task_string + + # Deletes a task from a TaskQueue. + elif request.method == "DELETE": + p = r.pipeline() + p.zrem(queue_key, task_id) + p.expire(name=task_id, time=120) + p.execute() + return "", 204 diff --git a/queueservice/docker-entrypoint.sh b/queueservice/docker-entrypoint.sh new file mode 100755 index 0000000..1b391ba --- /dev/null +++ b/queueservice/docker-entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/bash +export NETWORK_IPADDRESS=`awk 'END{print $1}' /etc/hosts` +echo 'export NETWORK_IPADDRESS=${NETWORK_IPADDRESS}' >> ~/.bashrc +/docker_venv/bin/gunicorn -c ./gunicorn.conf.py -b :${PORT} wsgi diff --git a/queueservice/gunicorn.conf.py b/queueservice/gunicorn.conf.py new file mode 100644 index 0000000..da8a443 --- /dev/null +++ b/queueservice/gunicorn.conf.py @@ -0,0 +1,10 @@ +"""Configuration file for gunicorn.""" +import multiprocessing + +# Details from https://cloud.google.com/appengine/docs/flexible/python/runtime +timeout = 120 +graceful_timeout = 60 +worker_class = 'gevent' +workers = multiprocessing.cpu_count() * 2 + 1 +forwarded_allow_ips = '*' +secure_scheme_headers = {'X-APPENGINE-HTTPS': 'on'} diff --git a/queueservice/requirements.txt b/queueservice/requirements.txt new file mode 100644 index 0000000..2199aed --- /dev/null +++ b/queueservice/requirements.txt @@ -0,0 +1,4 @@ +flask==0.12.1 +gevent==1.2.1 +gunicorn==19.7.1 +redis==2.10.5 diff --git a/queueservice/webserver.py b/queueservice/webserver.py new file mode 100644 index 0000000..a20a4b2 --- /dev/null +++ b/queueservice/webserver.py @@ -0,0 +1,74 @@ +"""Webserver for the queue service.""" +import os +import logging +import json +from flask import Flask +from api_data.taskqueue_v1beta2 import taskqueue_v1beta2_api + + +# FLASK SETUP + +HOST = os.getenv("NETWORK_IPADDRESS", "localhost") +PORT = int(os.getenv("PORT", 5052)) +application = Flask(__name__) +application.register_blueprint(taskqueue_v1beta2_api, url_prefix="/taskqueue/v1beta2/projects") + + +@application.route("/") +def index(): + """Give index page describing the service.""" + return "CS-Unplugged - Fake Google TaskQueue" + + +@application.route("/api//") +def api(api=None, version=None): + """Get an API description that is stored in data. + + The API will be a modified copy (usually to remove authorization + requirements) of the original from the mimicked source. + + Args: + api: The string of the api to load. (str) + version: The string of the version of the api to load. (str) + Returns: + A JSON object describing the API. + """ + content = None + filepath = os.path.join("api_data", "{0}_{1}.api".format(api, version)) + if not os.path.exists(filepath): + message = "API does not exist for {} version {}.".format(api, version) + logging.exception(message) + return message, 404 + elif not os.path.isfile(filepath): + message = "Server Error: API path exists for {} (version {}) but is not a file.".format(api, version) + logging.exception(message) + return message, 500 + with open(filepath, "r") as f: + api_json = json.loads(f.read()) + api_json["baseUrl"] = "http://{}:{}/{}/{}/projects/".format(HOST, PORT, api, version) + api_json["basePath"] = "/{}/{}/projects/".format(api, version) + api_json["rootUrl"] = "http://{}:{}/".format(HOST, PORT) + api_json["servicePath"] = "{}/{}/projects/".format(api, version) + content = json.dumps(api_json) + return content + + +@application.errorhandler(500) +def server_error(e): + """Log and reports back information about internal errors. + + Args: + e: The exception which was raised. (Exception) + Returns: + A string which describes the exception and the internal server + error status code. + """ + logging.exception("An error occurred during a request.") + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +if __name__ == "__main__": + application.run(debug=True, host="0.0.0.0", port=PORT) diff --git a/queueservice/wsgi.py b/queueservice/wsgi.py new file mode 100644 index 0000000..d60d9cd --- /dev/null +++ b/queueservice/wsgi.py @@ -0,0 +1,5 @@ +"""WSGI config for queue api service.""" +from webserver import application + +if __name__ == "__main__": + application.run() diff --git a/render b/render new file mode 100755 index 0000000..9759f16 --- /dev/null +++ b/render @@ -0,0 +1,238 @@ +#!/bin/bash +set -e + +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' # No Color + +cmd_helps=() +dev_cmd_helps=() + +defhelp() { + if [ "$1" = '-dev' ]; then + local command="${2?}" + local text="${3?}" + local help_str + help_str="$(printf ' %-18s %s' "$command" "$text")" + dev_cmd_helps+=("$help_str") + else + local command="${1?}" + local text="${2?}" + local help_str + help_str="$(printf ' %-18s %s' "$command" "$text")" + cmd_helps+=("$help_str") + fi +} + +# Print out help information +cmd_help() { + echo "Script for performing tasks related to the render service repository." + echo + echo "Usage: ./render [COMMAND]" + echo "Replace [COMMAND] with a word from the list below." + echo + echo "COMMAND list:" + for str in "${cmd_helps[@]}"; do + echo -e "$str" + done + echo + echo "DEV_COMMAND list:" + for str in "${dev_cmd_helps[@]}"; do + echo -e "$str" + done +} + +defhelp help 'View all help.' +defhelp 'dev [DEV_COMMAND]' 'Run a developer command.' + +# Start development environment +cmd_start() { + echo "Creating systems..." + docker-compose up -d + echo -e "\n${GREEN}Systems are ready!${NC}" +} +defhelp start 'Start development environment (this also runs the update command).' + +# Stop development environment +cmd_end() { + echo "Stopping systems... (takes roughly 10 to 20 seconds)" + docker-compose down + echo + echo "Deleting system volumes..." + docker volume ls -qf dangling=true | xargs -r docker volume rm +} +defhelp end 'Stop development environment.' + +# Restart development environment +cmd_restart() { + cmd_end + cmd_start +} +defhelp restart 'Stop and then restart development environment.' + +# Build Docker images +dev_build() { + echo "Building Docker images..." + docker-compose build + echo + echo "Deleting untagged images..." + docker images --no-trunc | grep '' | awk '{ print $3 }' | xargs -r docker rmi +} +defhelp -dev build 'Build or rebuild Docker images.' + +# Run shell +dev_shell() { + docker-compose exec render bash +} +defhelp -dev shell 'Open shell render service.' + +# Run style checks +dev_style() { + echo "Running PEP8 style checker..." + flake8 + pep8_status=$? + echo + echo "Running Python docstring checker..." + pydocstyle --count --explain + pydocstyle_status=$? + ! (( pep8_status || pydocstyle_status )) +} +defhelp -dev style 'Run style checks.' + +# Generates the documentation (with warnings as errors) +dev_docs() { + echo "Removing any existing documentation..." + rm -rf docs/build/ + mkdir docs/build/ + echo + echo "Creating documentation..." + sphinx-build -W docs/source/ docs/build/ +} +defhelp -dev docs 'Generate documentation.' + +# Run test suite +dev_test_suite() { + echo "Running test suite..." + docker-compose exec render /docker_venv/bin/coverage run --rcfile=.coveragerc -m render.tests.start_tests -v +} +defhelp -dev test 'Run test suite with code coverage.' + +# Display test coverage table +dev_test_coverage() { + echo "Displaying test suite coverage..." + docker-compose exec render /docker_venv/bin/coverage xml -i + docker-compose exec render /docker_venv/bin/coverage report -m --skip-covered +} +defhelp -dev test_coverage 'Display code coverage report.' + +dev_test_coverage_upload() { + echo "Uploading test suite coverage..." + docker cp "$(docker-compose ps -q render)":/renderservice/coverage.xml ./coverage.xml + bash <(curl -s https://codecov.io/bash) +} +defhelp -dev test_coverage_upload 'Upload coverage report to codecov.' + +# Delete all untagged dangling Docker images +cmd_clean() { + echo "If the following commands return an argument not found error," + echo "this is because there is nothing to delete for clean up." + + echo + echo "Deleting unused volumes..." + docker volume ls -qf dangling=true | xargs -r docker volume rm + echo + echo "Deleting exited containers..." + docker ps --filter status=dead --filter status=exited -aq | xargs docker rm -v + echo + echo "Deleting dangling images..." + docker images -f "dangling=true" -q | xargs docker rmi +} +defhelp clean 'Delete unused Docker files.' + +# Delete all Docker containers and images +cmd_wipe() { + docker ps -a -q | xargs docker rm + docker images -q | xargs docker rmi +} +defhelp wipe 'Delete all Docker containers and images.' + +# View logs +cmd_logs() { + docker-compose logs +} +defhelp logs 'View logs.' + +ci_test_with_coverage() { + dev_test_suite + test_status=$? + dev_test_coverage + coverage_status=$? + dev_test_coverage_upload + ! (( $test_status || $coverage_status )) +} + +ci_style() { + dev_style +} + +ci_docs() { + dev_docs +} + +silent() { + "$@" > /dev/null 2>&1 +} + +cmd_ci() { + cmd_start + local cmd="$1" + shift + if [ -z "$cmd" ]; then + echo -e "${RED}ERROR: ci command requires one parameter!${NC}" + cmd_help + exit 1 + fi + if silent type "ci_$cmd"; then + "ci_$cmd" "$@" + exit $? + else + echo -e "${RED}ERROR: Unknown command!${NC}" + echo "Type './csu help' for available commands." + return 1 + fi +} + +cmd_dev() { + local cmd="$1" + shift + if [ -z "$cmd" ]; then + echo -e "${RED}ERROR: dev command requires one parameter!${NC}" + cmd_help + return 1 + fi + if silent type "dev_$cmd"; then + "dev_$cmd" "$@" + exit $? + else + echo -e "${RED}ERROR: Unknown command!${NC}" + echo "Type './render help' for available commands." + return 1 + fi +} + +# If no command given +if [ $# -eq 0 ]; then + echo -e "${RED}ERROR: This script requires a command!${NC}" + cmd_help + exit 1 +fi +cmd="$1" +shift +if silent type "cmd_$cmd"; then + "cmd_$cmd" "$@" + exit $? +else + echo -e "${RED}ERROR: Unknown command!${NC}" + echo "Type './render help' for available commands." + exit 1 +fi diff --git a/renderservice/Dockerfile b/renderservice/Dockerfile new file mode 100644 index 0000000..882c8e0 --- /dev/null +++ b/renderservice/Dockerfile @@ -0,0 +1,68 @@ +# This Dockerfile is based off the Google App Engine Python runtime image +# https://github.com/GoogleCloudPlatform/python-runtime +FROM uccser/python:3.6.2-stretch-with-weasyprint + +# Add metadata to Docker image +LABEL maintainer="csse-education-research@canterbury.ac.nz" + +# Set terminal to be noninteractive +ARG DEBIAN_FRONTEND=noninteractive +ARG PROCESS_PORT=8080 +ARG FLASK_PRODUCTION=1 +ARG PROJECT_NAME=cs-unplugged +ARG QUEUE_NAME=render-queue +ARG API_DISCOVERY_URL +ARG BUCKET_NAME=cs-unplugged-dev.appspot.com +ARG STATIC_DIRECTORY=/renderservice/static_mnt + +ENV PORT ${PROCESS_PORT} +ENV FLASK_PRODUCTION ${FLASK_PRODUCTION} +ENV PROJECT_NAME ${PROJECT_NAME} +ENV QUEUE_NAME ${QUEUE_NAME} +ENV API_DISCOVERY_URL ${API_DISCOVERY_URL} +ENV STATIC_DIRECTORY ${STATIC_DIRECTORY} +ENV CLOUD_STORAGE_BUCKET_NAME ${BUCKET_NAME} +ENV GOOGLE_CLOUD_BUCKET_KEY /keys/bucket.json +ENV GOOGLE_APPLICATION_CREDENTIALS /keys/compute.json + +# Install GCSFUSE +RUN apt-get update \ + && apt-get install -y lsb-release kmod \ + && export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s` \ + && echo "deb http://packages.cloud.google.com/apt $GCSFUSE_REPO main" | tee /etc/apt/sources.list.d/gcsfuse.list \ + && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ + && apt-get update \ + && apt-get install -y \ + gcsfuse \ + && apt-get clean && rm /var/lib/apt/lists/*_* + +# Set-up file system +RUN mkdir /renderservice +RUN mkdir /renderservice/scripts +RUN mkdir /keys +COPY requirements.txt /renderservice +COPY bucket.json /keys/bucket.json +COPY compute.json /keys/compute.json + +# Install dependencies +RUN /docker_venv/bin/pip3 install -r /renderservice/requirements.txt + +# Setup Working Directory +ADD . /renderservice +WORKDIR /renderservice + +# Setup User +RUN useradd -c "RenderService user" -m -d /home/render -s /bin/bash render \ + && chown -R render /renderservice \ + && mkdir ${STATIC_DIRECTORY} \ + && chown -R render ${STATIC_DIRECTORY} +USER render +ENV HOME /home/render + +EXPOSE ${PORT} + +# Run System +RUN chmod +x /renderservice/scripts/mount-bucket.sh +RUN chmod +x /renderservice/scripts/docker-entrypoint.sh +RUN chmod +x /renderservice/scripts/deploy-docker-entrypoint.sh +ENTRYPOINT ["/renderservice/scripts/deploy-docker-entrypoint.sh"] diff --git a/renderservice/Dockerfile-local b/renderservice/Dockerfile-local new file mode 100644 index 0000000..9b5c7b5 --- /dev/null +++ b/renderservice/Dockerfile-local @@ -0,0 +1,49 @@ +# This Dockerfile is based off the Google App Engine Python runtime image +# https://github.com/GoogleCloudPlatform/python-runtime +FROM uccser/python:3.6.2-stretch-with-weasyprint + +# Add metadata to Docker image +LABEL maintainer="csse-education-research@canterbury.ac.nz" + +# Set terminal to be noninteractive +ARG DEBIAN_FRONTEND=noninteractive +ARG PROCESS_PORT=8080 +ARG FLASK_PRODUCTION=0 +ARG PROJECT_NAME=cs-unplugged +ARG QUEUE_NAME=render-queue +ARG API_DISCOVERY_URL +ARG STATIC_DIRECTORY=/renderservice/static_mnt + +ENV PORT ${PROCESS_PORT} +ENV FLASK_PRODUCTION ${FLASK_PRODUCTION} +ENV PROJECT_NAME ${PROJECT_NAME} +ENV QUEUE_NAME ${QUEUE_NAME} +ENV API_DISCOVERY_URL ${API_DISCOVERY_URL} +ENV STATIC_DIRECTORY ${STATIC_DIRECTORY} + +# Set-up file system +RUN mkdir /renderservice +RUN mkdir /renderservice/scripts +COPY ./renderservice/requirements.txt /renderservice + +# Install dependencies +RUN /docker_venv/bin/pip3 install -r /renderservice/requirements.txt + +# Setup Working Directory +ADD ./renderservice /renderservice +WORKDIR /renderservice +COPY .coveragerc /renderservice/ + +# Setup User +RUN useradd -c "RenderService user" -m -d /home/render -s /bin/bash render \ + && chown -R render.render /renderservice \ + && mkdir ${STATIC_DIRECTORY} \ + && chown -R render.render ${STATIC_DIRECTORY} +USER render +ENV HOME /home/render + +EXPOSE ${PORT} + +# Run System +RUN chmod +x /renderservice/scripts/docker-entrypoint.sh +ENTRYPOINT ["/renderservice/scripts/docker-entrypoint.sh"] diff --git a/renderservice/render/__init__.py b/renderservice/render/__init__.py new file mode 100644 index 0000000..3b66779 --- /dev/null +++ b/renderservice/render/__init__.py @@ -0,0 +1,2 @@ +"""Render Service containing both Daemons and Webserver.""" +__version__ = "1.0.0-alpha.1" diff --git a/renderservice/render/daemon/FileManager.py b/renderservice/render/daemon/FileManager.py new file mode 100644 index 0000000..06f58bb --- /dev/null +++ b/renderservice/render/daemon/FileManager.py @@ -0,0 +1,95 @@ +"""Handles access to external filesystems from the render service.""" +import os +from cachetools import LRUCache, cachedmethod +from io import BytesIO + +CACHE_SIZE = int(os.getenv("DAEMON_CACHE_SIZE", 300 * 1024 * 1024)) + + +class FileManager(object): + """Handles access to the filesystem for loading and reading files. + + Caching is important to this class as it speeds up slow reads when + the directory is a mounted bucket. + + Since files with the same filepath could belong to different + directories only the first found is used, based on the order + of the constructor arguments in FIFO. + """ + + __cache = LRUCache(maxsize=CACHE_SIZE, getsizeof=lambda value: value.__sizeof__()) + + def __init__(self, *args, **kwargs): + """Create a resource manager. + + Args: + args: Directories to load files from. (list of strings) + """ + if len(args) == 0: + raise Exception() # TODO + self.directories = tuple(args) + self.save_directory = kwargs.get("save_directory", None) + + @cachedmethod(lambda cls: type(cls).__cache) + def load(self, filepath): + """Load a file from the directory/mounted cloud bucket. + + Args: + filepath: The name of the file including full path to the + file within the storage bucket. (str) + Returns: + BytesIO stream of the file contents. (BytesIO) + """ + path = self.get_path(filepath) + data = None + with open(path, 'rb') as f: + data = f.read() + return BytesIO(data) + + def get_path(self, filepath): + """Get the full filepath for a given resource. + + Finds the first file that matches the filepath (this means + if a folder matches first it is not returned.) + + Args: + filepath: The name of the file including full path to the + file within the storage bucket. (str) + Returns: + A string of the filepath. (str) + """ + for directory in self.directories: + path = os.path.join(directory, filepath) + if os.path.exists(path) and os.path.isfile(path): + return path + raise OSError("File not found.") + + def save(self, filepath, content): + """Save a file to the cloud bucket. + + If save directory is not set, tries to find the file within + any directories contained to overwrite it. If no directory + is found then writes to the first given directory. + + Args: + filepath: The name of the file including full path to the + file within the storage bucket. (str) + content: Bytes of the content to be saved to the file. (bytes) + """ + save_path = None + if self.save_directory is None: + for directory in self.directories: + path = os.path.join(directory, filepath) + if os.path.exists(path) and os.path.isfile(path): + save_path = path + + if save_path is None: + directory = self.directories[0] + save_path = os.path.join(directory, filepath) + else: + save_path = os.path.join(self.save_directory, filepath) + + save_directory, _ = os.path.split(save_path) + os.makedirs(save_directory, exist_ok=True) + with open(save_path, "wb") as f: + f.write(content) diff --git a/renderservice/render/daemon/QueueHandler.py b/renderservice/render/daemon/QueueHandler.py new file mode 100644 index 0000000..74fee62 --- /dev/null +++ b/renderservice/render/daemon/QueueHandler.py @@ -0,0 +1,231 @@ +"""Handles transactions with the taskqueue api.""" +import json +import logging +import httplib2shim +from apiclient.discovery import build, HttpError +from base64 import b64encode, b64decode + +logger = logging.getLogger(__name__) + + +def authorize_session(): + """Authorize for taskqueue transactions. + + Returns: + Something in future! + """ + pass # TODO + # Should probably just use https://developers.google.com/identity/protocols/application-default-credentials + from oauth2client.client import GoogleCredentials + credentials = GoogleCredentials.get_application_default() + return credentials + + +def encode_dictionary(dictionary): + """Encode a dictionary into a base64 string. + + Args: + dictionary: A python dictionary to convert. (dict) + Returns: + The encoded string. (base64 string) + """ + string = json.dumps(dictionary) + encoded_string = b64encode(string.encode("ascii")).decode() + return encoded_string + + +def decode_dictionary(encoded_string): + """Decode a base64 string into a dictionary. + + Args: + encoded_string: A base64 string to decode. (base64 string) + Returns: + A python dictionary deserialized from the string. (dict) + """ + string = b64decode(encoded_string).decode() + dictionary = json.loads(string) + return dictionary + + +class QueueHandler(object): + """Handles transactions with the taskqueue api.""" + + def __init__(self, project_name, taskqueue_name, discovery_url=None): + """Create a new QueueHandler. + + Args: + project_name: The project the taskqueue belongs to. (str) + taskqueue_name: The name of the taskqueue. (str) + """ + self.project_name = project_name + self.taskqueue_name = taskqueue_name + + http = httplib2shim.Http() + if discovery_url is not None: + self.task_api = build("taskqueue", "v1beta2", http=http, discoveryServiceUrl=discovery_url) + else: + self.task_api = build("taskqueue", "v1beta2", http=http) + + def __len__(self): + """Count the number of tasks within the queue.""" + try: + get_request = self.task_api.taskqueues().get( + project=self._get_project_name(False), + taskqueue=self.taskqueue_name, + getStats=True + ) + result = get_request.execute() + return result["stats"]["totalTasks"] + except HttpError as http_error: + logger.error("Error during get request: {}".format(http_error)) + return 0 + + def _get_project_name(self, is_write): + """Get the project name based for write command. + + Args: + is_write: A boolean determining if the name will be used + for a write operation. (bool) + Returns: + A string of the project name required for a task_api call. (str) + """ + if is_write: + return "b~" + self.project_name + return self.project_name + + def list_tasks(self): + """List some tasks within the taskqueue. + + Returns: + A list of Google Tasks as with the user defined + task (dictionary) under that 'payload' key. (list of dicts) + """ + try: + tasks = [] + list_request = self.task_api.tasks().list( + project=self._get_project_name(False), + taskqueue=self.taskqueue_name + ) + result = list_request.execute() + if result["kind"] == "taskqueue#tasks": + for task in result["items"]: + task["payload"] = decode_dictionary(task["payloadBase64"]) + tasks.append(task) + elif result["kind"] == "taskqueues#task": + task["payload"] = decode_dictionary(result["payloadBase64"]) + tasks.append(task) + return tasks + except HttpError as http_error: + logger.error("Error during lease request: {}".format(http_error)) + return [] + + def create_task(self, task_payload, tag=None): + """Create a new task and places it on the taskqueue. + + Args: + task_payload: A dictionary describing the task. (dict) + tag: A tag attached to the task. (str) + Returns: + The task id of the created task, otherwise None if error. (str) + """ + try: + task = { + "kind": "taskqueues#task", + "queueName": self.taskqueue_name, + "payloadBase64": encode_dictionary(task_payload) + } + if tag is not None: + task["tag"] = tag + + insert_request = self.task_api.tasks().insert( + project=self._get_project_name(True), + taskqueue=self.taskqueue_name, + body=task + ) + result = insert_request.execute() + return result["id"] + except HttpError as http_error: + logger.error("Error during insert request: {}".format(http_error)) + return None + + def lease_tasks(self, tasks_to_fetch, lease_secs, tag=None): + """Lease tasks from the taskqueue. + + Args: + tasks_to_fetch: The number of tasks to fetch. (int) + lease_secs: The number of seconds to lease for. (int) + tag: the tag to restrict leasing too. (str) + Returns: + A list of Google Tasks as with the user defined + task (dictionary) under that 'payload' key. (list of dicts) + """ + try: + tasks = [] + lease_request = self.task_api.tasks().lease( + project=self._get_project_name(True), + taskqueue=self.taskqueue_name, + leaseSecs=lease_secs, + numTasks=tasks_to_fetch, + groupByTag=tag is not None, + tag=tag + ) + result = lease_request.execute() + if result["kind"] == "taskqueue#tasks": + for task in result["items"]: + task["payload"] = decode_dictionary(task["payloadBase64"]) + tasks.append(task) + elif result["kind"] == "taskqueues#task": + task["payload"] = decode_dictionary(result["payloadBase64"]) + tasks.append(task) + return tasks + except HttpError as http_error: + logger.error("Error during lease request: {}".format(http_error)) + return [] + + def update_task(self, task_id, new_lease_secs): + """Update a task lease from the taskqueue. + + Args: + task_id: A string of the task_id. (str) + new_lease_secs: The number of seconds to update the lease + by. (int) + Returns: + The updated Google Task as a dictionary, the payload is + untouched. If there is an error None is returned. (dict) + """ + try: + task = { + "queueName": self.taskqueue_name + } + patch_request = self.task_api.tasks().patch( + project=self._get_project_name(True), + taskqueue=self.taskqueue_name, + newLeaseSeconds=new_lease_secs, + task=task_id, + body=task + ) + result = patch_request.execute() + return result + except HttpError as http_error: + logger.error("Error during lease request: {}".format(http_error)) + return None + + def delete_task(self, task_id): + """Delete a task from the taskqueue. + + Args: + task_id: A string of the task_id. (str) + Returns: + True if the delete was successful, False otherwise. (bool) + """ + try: + delete_request = self.task_api.tasks().delete( + project=self._get_project_name(True), + taskqueue=self.taskqueue_name, + task=task_id + ) + delete_request.execute() + return True + except HttpError as http_error: + logger.error("Error during delete request: {}".format(http_error)) + return False diff --git a/renderservice/render/daemon/RenderDaemon.py b/renderservice/render/daemon/RenderDaemon.py new file mode 100644 index 0000000..97ab775 --- /dev/null +++ b/renderservice/render/daemon/RenderDaemon.py @@ -0,0 +1,200 @@ +"""Render Daemon for collecting and consuming render jobs.""" +import os +import sys +import time +import signal +import logging +from io import BytesIO +from base64 import b64encode +from daemons.prefab.run import RunDaemon +from render.daemon.QueueHandler import QueueHandler +from render.daemon.ResourceGenerator import ResourceGenerator + +# Daemon Setup and Task Management Constants +PROJECT_NAME = os.getenv("PROJECT_NAME", None) +QUEUE_NAME = os.getenv("QUEUE_NAME", None) +DISCOVERY_URL = os.getenv("API_DISCOVERY_URL", None) + +TASK_COUNT = int(os.getenv("TASK_COUNT", 20)) +TASK_SECONDS = float(os.getenv("TASK_SECONDS", 50)) +TASK_TIME_MULT = float(os.getenv("TASK_TIME_MULT", 1.33)) +TASK_RETRY_LIMIT = int(os.getenv("TASK_RETRY_LIMIT", 5)) + +RENDER_SLEEP_TIME = float(os.getenv("RENDER_SLEEP_TIME", 10)) +MAX_QUEUE_TASK_SIZE = int(os.getenv("MAX_QUEUE_TASK_SIZE", 680 * 1024)) + +CLOUD_STORAGE_BUCKET_NAME = os.getenv("CLOUD_STORAGE_BUCKET_NAME", "cs-unplugged-ev.appspot.com") +BUCKET_SAVE_DIRECTORY = os.getenv("BUCKET_SAVE_DIRECTORY", "/static/resources") + +logger = logging.getLogger(__name__) + + +def authenticate_storage(): + """Authenticate into Google Cloud storage. + + Returns: + An authenticate storage client. (Client) + """ + from google.auth import compute_engine + from google.cloud import storage + + credentials = compute_engine.Credentials() + storage_client = storage.Client(credentials=credentials, project=PROJECT_NAME) + + return storage_client + + +def handle_timelimit_exceeded(): + """Raise the timeout exception when SIGALRM signal is caught.""" + raise TimeoutError("Timelimit exceeded.") + + +class RenderDaemon(RunDaemon, ResourceGenerator): + """A daemon that processes tasks related to the rendering pipeline. + + WARN: Be careful here you understand which task you are dealing + with, a google task or one of our tasks. + """ + + def __init__(self, *args, **kwargs): + """Create a Render Daemon. + + Assumes that any SIGALRM signals are sent by itself for + timeout exceptions. + """ + super(RenderDaemon, self).__init__(*args, **kwargs) + self.handle(signal.SIGALRM, handle_timelimit_exceeded) + # Handle SIGUSR1 for closing up for pre-emption. + + def run(self): + """Consumes jobs and produces rendered documents.""" + queue = QueueHandler(project_name=PROJECT_NAME, taskqueue_name=QUEUE_NAME, discovery_url=DISCOVERY_URL) + logger.info("Daemon with pid {} running.".format(self.pid)) + while True: + lease_secs = TASK_COUNT * TASK_SECONDS + tasks = queue.lease_tasks(tasks_to_fetch=TASK_COUNT, lease_secs=lease_secs, tag="task") + self.process_tasks(tasks, queue) + time.sleep(RENDER_SLEEP_TIME) + + def process_tasks(self, tasks, queue): + """Run main loop for determining individual task logic. + + Tasks will be run to recieve a result, saved if necessary + then deleted. Tasks that return a none result or are + interupted by an exception will not be deleted and have + their lease cleared for another daemon. Tasks that have + surpassed their retry limit will have a failure result + saved and be deleted. + + Args: + tasks: A list of json task objects. (list of dicts) + queue: QueueHandler to update and delete tasks from. (str) + """ + for task_descriptor in tasks: + task_id = task_descriptor["id"] + retries = task_descriptor["retry_count"] + timeout_seconds = int(TASK_SECONDS + TASK_SECONDS * TASK_TIME_MULT * retries) + + result = None + if retries < TASK_RETRY_LIMIT: + signal.alarm(timeout_seconds) + try: + result = self.process_task(task_descriptor) + except Exception as e: + logger.exception("Task {} raised exception with error: {}".format(task_descriptor["id"], e)) + finally: + signal.alarm(0) + else: + result = self.handle_retry_limit(task_descriptor) + + # Save documents + if result is not None and result["kind"] == "result#document": + if MAX_QUEUE_TASK_SIZE < sys.getsizeof(result["document"]): + filename = result["filename"] + document = result["document"] + public_url = self.handle_document_saving(filename, document) + link_result = { + "kind": "result#link", + "success": result["success"], + "url": public_url + } + result = link_result + + queue.create_task(task_payload=result, tag="result") + + # Task was successful or had too many failures + if result is not None: + queue.delete_task(task_id) + # Task failed and should be retried + else: + queue.update_task(task_id=task_id, new_lease_secs=1) + + def process_task(self, task_descriptor): + """Process the given task and get result. + + Render tasks produce and save out documents. + + Args: + task_descriptor: The queue task with the user + definied task as the payload. (dict) + Returns: + A dictionary of the result. (dict) + """ + task = task_descriptor["payload"] + task_kind = task["kind"] + result = None + + if task_kind == "task#render": + filename, document = self.generate_resource_pdf(task) + result = { + "kind": "result#document", + "success": True, + "filename": filename, + "document": b64encode(document).decode("ascii") + } + else: + raise Exception("Unrecognized task: {}.".format(task_kind)) + + return result + + def handle_retry_limit(self, task_descriptor): + """Process the given task and get result. + + Render tasks produce and save out documents. + + Args: + task_descriptor: The queue task with the user + definied task as the payload. (dict) + Returns: + A dictionary of the result. (dict) + """ + task = task_descriptor["payload"] + task_kind = task["kind"] + result = dict() # result should never be None + + if task_kind == "task#render": + result = { + "kind": "result#document", + "success": False, + "filename": None, + "document": None + } + return result + + def handle_document_saving(self, filename, document): + """Save a given document to the google cloud bucket. + + Args: + filename: A string of the name to save the file as within + the bucket. (str) + document: Bytes of the document to be saved. (bytes) + Returns: + A public url to the document. (str) + """ + client = authenticate_storage() + bucket = client.get_bucket(CLOUD_STORAGE_BUCKET_NAME) + blob = bucket.blob(os.path.join(BUCKET_SAVE_DIRECTORY, filename)) + blob.make_public() + file_stream = BytesIO(document) + blob.upload_from_file(file_stream) + return blob.public_url diff --git a/renderservice/render/daemon/ResourceGenerator.py b/renderservice/render/daemon/ResourceGenerator.py new file mode 100644 index 0000000..5f6770b --- /dev/null +++ b/renderservice/render/daemon/ResourceGenerator.py @@ -0,0 +1,169 @@ +"""Render Daemon for collecting and consuming render jobs.""" +import os +import logging +import importlib +from io import BytesIO +from PIL import Image +from base64 import b64encode +from jinja2 import Environment, FileSystemLoader +from weasyprint import HTML, CSS +from render.daemon.FileManager import FileManager + +# Daemon Setup and Task Management Constants +STATIC_DIRECTORY = os.getenv("STATIC_DIRECTORY", "/renderservice/static_mnt") +TEMPLATE_DIRECTORY = os.getenv("TEMPLATE_DIRECTORY", "/renderservice/templates") + +logger = logging.getLogger(__name__) + +# File Generation and Processing Constants +MM_TO_PIXEL_RATIO = 6 +A4_MM_SCALE = 267 +LETTER_MM_SCALE = 249 + + +class TaskError(Exception): + """An error associated with a malformed task object.""" + + def __init__(self, *args, **kwargs): + """Create a TaskError.""" + super(TaskError, self).__init__(*args, **kwargs) + + +class ResourceGenerator(object): + """Turns a task into a pdf or image.""" + + def __init__(self, *args, **kwargs): + """Create a Render Daemon. + + Assumes that any SIGALRM signals are sent by itself for + timeout exceptions. + """ + super(ResourceGenerator, self).__init__(*args, **kwargs) + self.file_manager = FileManager("/renderservice/static", STATIC_DIRECTORY, save_directory=STATIC_DIRECTORY) + self.template_environment = Environment( + loader=FileSystemLoader(TEMPLATE_DIRECTORY), + autoescape=False + ) + + def import_resource_module(self, resource_view): + """Get the resource specification. + + Args: + resource_view: A string of the python filename. (str) + Returns: + A python module. (module) + """ + if resource_view.endswith(".py"): + resource_view = resource_view[:-3] + module_path = "render.resources.{}".format(resource_view) + return importlib.import_module(module_path) + + def generate_resource_pdf(self, task): + """Return a response containing a generated PDF resource. + + Args: + task: A dicitionary of values specifying the task. + Must have: + - resource_slug + - resource_name + - resource_view + - header_text (optional) + - paper_size + - copies + - url + + Returns: + Tuple of filename and PDF file of generated resource. (tuple) + """ + if task.get("resource_slug", None) is None: + raise TaskError("Task must specify the resource slug.") + if task.get("resource_name", None) is None: + raise TaskError("Task must specify the resource name.") + if task.get("resource_view", None) is None: + raise TaskError("Task must specify the resource view.") + if task.get("paper_size", None) is None: + raise TaskError("Task must specify paper size.") + if task.get("copies", None) is None: + raise TaskError("Task must specify number of copies.") + if task.get("url", None) is None: + raise TaskError("Task must specify the url.") + + resource_generator = self.import_resource_module(task["resource_view"]) + + for option, values in resource_generator.valid_options().items(): + if option not in task.keys(): + raise TaskError("Task is missing value for {}.".format(option)) + if task[option] not in values: + raise TaskError("Value ({}) for option {} is not in: {}.".format(task[option], option, values)) + + context = dict() + context["resource_name"] = task["resource_name"] + context["header_text"] = task.get("header_text", "") + context["paper_size"] = task["paper_size"].lower() + context["url"] = task["url"] + + context["all_data"] = [] + for copy in range(0, task["copies"]): + context["all_data"].append( + self.generate_resource(task, resource_generator) + ) + + filename = "{} ({}).pdf".format(task["resource_name"], resource_generator.subtitle(task)) + context["filename"] = filename + + template_filename = task.get("template", "base-resource-pdf.html") + css_filename = task.get("css", "css/print-resource-pdf.css") + + template = self.template_environment.get_template(template_filename) + pdf_html = template.render(context) # TODO: Future consider async + html = HTML(string=pdf_html, base_url=STATIC_DIRECTORY) + css_data = self.file_manager.load(css_filename).read() + css_string = css_data.decode("utf-8") + logger.info(css_string) + base_css = CSS(string=css_string) + return filename, html.write_pdf(stylesheets=[base_css]) + + def generate_resource(self, task, resource_generator): + """Retrieve page(s) for one copy of resource from resource generator. + + Images are resized to fit page. + + Args: + task: The specification of file to generate as a dictionary. (dict) + resource_generator: The file generation module. (module) + + Returns: + List of Base64 strings of a generated resource images for one copy. (list of base64 strings) + """ + # Get images from resource image creator + data = resource_generator.resource(task, self.file_manager) + if not isinstance(data, list): + data = [data] + + # Resize images to reduce file size + max_pixel_height = 0 + if task["paper_size"].lower() == "a4": + max_pixel_height = A4_MM_SCALE * MM_TO_PIXEL_RATIO + elif task["paper_size"].lower() == "letter": + max_pixel_height = LETTER_MM_SCALE * MM_TO_PIXEL_RATIO + else: + raise TaskError("Unsupported paper size: {}.".format(task["paper_size"])) + + for index in range(len(data)): + if data[index]["type"] == "image": + image = data[index]["data"] + width, height = image.size + if height > max_pixel_height: + ratio = max_pixel_height / height + width *= ratio + height *= ratio + image = image.resize((int(width), int(height)), Image.ANTIALIAS) + + # Save image to buffer + image_buffer = BytesIO() + image.save(image_buffer, format="PNG") + + # Add base64 of image to list of images + data[index]["data"] = b64encode(image_buffer.getvalue()).decode() + + return data diff --git a/renderservice/render/daemon/__init__.py b/renderservice/render/daemon/__init__.py new file mode 100644 index 0000000..f55ab2e --- /dev/null +++ b/renderservice/render/daemon/__init__.py @@ -0,0 +1 @@ +"""Module containing daemon logic for consuming and rendering tasks.""" diff --git a/renderservice/render/daemon/__main__.py b/renderservice/render/daemon/__main__.py new file mode 100644 index 0000000..62bd5db --- /dev/null +++ b/renderservice/render/daemon/__main__.py @@ -0,0 +1,89 @@ +"""Module containing daemon logic for consuming and rendering tasks.""" +import os +import sys +import logging +import optparse +from logging.handlers import RotatingFileHandler +from render.daemon.utils import PID_DIRECTORY +from render.daemon.RenderDaemon import RenderDaemon + +LOG_DIRECTORY = os.getenv("DAEMON_LOG_DIRECTORY", os.path.join(os.getcwd(), "logs")) + + +def parse_args(): + """Command-line option parser for program control. + + For usage & options, type: "render.py -h". + """ + opts = optparse.OptionParser( + usage="{0} [options] input-data-set(s)".format(sys.argv[0]), + description="Create, modify and kill render daemons.") + opts.add_option("--daemon", + "-d", + action="store", + type="int", + help="The number of the daemon to apply the command too.", + default=None) + options, arguments = opts.parse_args() + return options, arguments + + +def setup_logging(options): + """Initialise the logging configuration. + + Args: + options: The program options. (optparse options) + """ + max_log_size = 100 * 1024 * 1024 + logs_directory = LOG_DIRECTORY + os.makedirs(logs_directory, exist_ok=True) + + logfile = os.path.join(logs_directory, "render_{}.log".format(options.daemon)) + log_formatter = logging.Formatter("%(asctime)-20s %(levelname)s:%(name)-30s Message: %(message)s") + + log_handler = RotatingFileHandler(logfile, + mode="a", + maxBytes=max_log_size, + backupCount=5, + encoding=None, + delay=False) + log_handler.setFormatter(log_formatter) + log_handler.setLevel(logging.INFO) + + render_log = logging.getLogger() + render_log.setLevel(logging.INFO) + render_log.addHandler(log_handler) + + +def render_daemon_control(daemon, action): + """Load and request a daemon to perform an action. + + Do not trust code or the process to be running after this method + as certain actions such as 'start' cause the main thread to be + killed. + + Args: + daemon: An integer representing the daemon number. (int) + action: A string which is either 'start', 'stop', 'restart' (str) + """ + # Set-up directories + pid_directory = PID_DIRECTORY + os.makedirs(pid_directory, exist_ok=True) + pidfile = os.path.join(pid_directory, "render_{}.pid".format(daemon)) + + d = RenderDaemon(pidfile=pidfile) + + if action == "start": + d.start() + elif action == "stop": + d.stop() + elif action == "restart": + d.restart() + + +if __name__ == "__main__": + options, arguments = parse_args() + setup_logging(options) + action = arguments[0] + + render_daemon_control(options.daemon, action) diff --git a/renderservice/render/daemon/utils.py b/renderservice/render/daemon/utils.py new file mode 100644 index 0000000..8240acc --- /dev/null +++ b/renderservice/render/daemon/utils.py @@ -0,0 +1,75 @@ +"""Utility helper functions for working with daemons.""" +import os +import re +import multiprocessing +from collections import namedtuple + +PID_DIRECTORY = os.getenv("PID_DIRECTORY", os.path.join(os.getcwd(), "pidstore")) + + +def check_pid(pid): + """Check that process is still active. + + Args: + pid: The process id of the process. (int) + Returns: + True if the process exists and is active, False otherwise. (bool) + """ + try: + os.kill(pid, 0) # kill actually means send UNIX signal. + except OSError: + return False + else: + return True + + +def get_active_daemon_details(daemon): + """Get the pids of all render daemons. + + Args: + daemon: A string of the daemon type. E.g. render. (str) + Returns: + An array of namedtuples containing daemon number to pid. (list of namedtuples) + """ + if not os.path.exists(PID_DIRECTORY): + return [] + + DaemonMetaData = namedtuple("DaemonMetaData", "number, pid") + regex = re.compile(r"^{}_(?P\d*).pid$".format(daemon)) + + details = [] + for filename in os.listdir(PID_DIRECTORY): + m = regex.match(filename) + if m is not None: + filepath = os.path.join(PID_DIRECTORY, filename) + with open(filepath, 'r') as f: + pid = int(f.read()) + number = int(m.group('number')) + details.append(DaemonMetaData(number, pid)) + return details + + +def get_recommended_number_of_daemons(): + """Get the recommended number of daemons to run on system. + + Returns: + An integer of the number of daemons. (int) + """ + try: + m = None + with open('/proc/self/status') as f: + m = re.search(r'(?m)^Cpus_allowed:\s*(.*)$', f.read()) + if m: + threads = int(m.group(1).replace(',', ''), 16) + res = bin(threads).count('1') + if res > 0: + return res + except IOError: + pass + + try: + return multiprocessing.cpu_count() + except (ImportError, NotImplementedError): + pass + + return 1 diff --git a/renderservice/render/resources/arrows.py b/renderservice/render/resources/arrows.py new file mode 100644 index 0000000..49752b0 --- /dev/null +++ b/renderservice/render/resources/arrows.py @@ -0,0 +1,48 @@ +"""Module for generating Arrows resource.""" + +from PIL import Image, ImageDraw + + +def resource(task, resource_manager): + """Create a copy of the Arrows resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dicitonaries for each resource page. + """ + image_path = "img/resources/arrows/arrows.png" + data = resource_manager.load(image_path) + image = Image.open(data) + ImageDraw.Draw(image) + return {"type": "image", "data": image} + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + Text for subtitle. (str) + """ + return task["paper_size"] + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "paper_size": ["a4", "letter"] + } diff --git a/renderservice/render/resources/barcode_checksum_poster.py b/renderservice/render/resources/barcode_checksum_poster.py new file mode 100644 index 0000000..d7a400b --- /dev/null +++ b/renderservice/render/resources/barcode_checksum_poster.py @@ -0,0 +1,51 @@ +"""Module for generating Barcode Checksum Poster resource.""" + +from PIL import Image + + +def resource(task, resource_manager): + """Create a image for Barcode Checksum Poster resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + barcode_length = task["barcode_length"] + image_path = "img/resources/barcode-checksum-poster/{}-digits.png" + data = resource_manager.load(image_path.format(barcode_length)) + image = Image.open(data) + return {"type": "image", "data": image} + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + text for subtitle. (str) + """ + barcode_length = task["barcode_length"] + paper_size = task["paper_size"] + return "{} digits - {}".format(barcode_length, paper_size) + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "barcode_length": ["12", "13"], + "paper_size": ["a4", "letter"], + } diff --git a/renderservice/render/resources/binary_cards.py b/renderservice/render/resources/binary_cards.py new file mode 100644 index 0000000..fcebb2a --- /dev/null +++ b/renderservice/render/resources/binary_cards.py @@ -0,0 +1,113 @@ +"""Module for generating Binary Cards resource.""" + +import os.path +from PIL import Image, ImageDraw, ImageFont + + +def resource(task, resource_manager): + """Create a image for Binary Cards resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + BASE_IMAGE_PATH = "img/resources/binary-cards/" + IMAGE_SIZE_X = 2480 + IMAGE_SIZE_Y = 3508 + IMAGE_DATA = [ + ("binary-cards-1-dot.png", 1), + ("binary-cards-2-dots.png", 2), + ("binary-cards-4-dots.png", 4), + ("binary-cards-8-dots.png", 8), + ("binary-cards-16-dots.png", 16), + ("binary-cards-32-dots.png", 32), + ("binary-cards-64-dots.png", 64), + ("binary-cards-128-dots.png", 128), + ] + + # Retrieve parameters + display_numbers = task["display_numbers"] + black_back = task["black_back"] + + if display_numbers: + font_path = "fonts/PatrickHand-Regular.ttf" + local_font_path = resource_manager.get_path(font_path) + font = ImageFont.truetype(local_font_path, 600) + BASE_COORD_X = IMAGE_SIZE_X / 2 + BASE_COORD_Y = IMAGE_SIZE_Y - 100 + IMAGE_SIZE_Y = IMAGE_SIZE_Y + 300 + + pages = [] + for (image_path, number) in IMAGE_DATA: + data = resource_manager.load(os.path.join(BASE_IMAGE_PATH, image_path)) + image = Image.open(data) + if display_numbers: + background = Image.new("RGB", (IMAGE_SIZE_X, IMAGE_SIZE_Y), "#FFF") + background.paste(image, mask=image) + draw = ImageDraw.Draw(background) + text = str(number) + text_width, text_height = draw.textsize(text, font=font) + coord_x = BASE_COORD_X - (text_width / 2) + coord_y = BASE_COORD_Y - (text_height / 2) + draw.text( + (coord_x, coord_y), + text, + font=font, + fill="#000" + ) + image = background + pages.append({"type": "image", "data": image}) + + if black_back: + black_card = Image.new("1", (IMAGE_SIZE_X, IMAGE_SIZE_Y)) + pages.append({"type": "image", "data": black_card}) + + return pages + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + text for subtitle. (str) + """ + if task["display_numbers"]: + display_numbers_text = "with numbers" + else: + display_numbers_text = "without numbers" + + if task["black_back"]: + black_back_text = "with black back" + else: + black_back_text = "without black back" + + text = "{} - {} - {}".format( + display_numbers_text, + black_back_text, + task["paper_size"] + ) + return text + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "display_numbers": [True, False], + "black_back": [True, False], + "paper_size": ["a4", "letter"], + } diff --git a/renderservice/render/resources/binary_cards_small.py b/renderservice/render/resources/binary_cards_small.py new file mode 100644 index 0000000..d03815e --- /dev/null +++ b/renderservice/render/resources/binary_cards_small.py @@ -0,0 +1,114 @@ +"""Module for generating Binary Cards (Small) resource.""" + +import os.path +from PIL import Image, ImageDraw, ImageFont + + +def resource(task, resource_manager): + """Create a image for Binary Cards (Small) resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (TaskManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + BASE_IMAGE_PATH = "img/resources/binary-cards-small/" + IMAGE_SIZE_X = 2480 + IMAGE_SIZE_Y = 3044 + IMAGE_DATA = [ + ("binary-cards-small-1.png", 4), + ("binary-cards-small-2.png", 8), + ("binary-cards-small-3.png", 12), + ] + + # Retrieve parameters + requested_bits = task["number_bits"] + dot_counts = task["dot_counts"] + black_back = task["black_back"] + + if dot_counts: + font_path = "fonts/PatrickHand-Regular.ttf" + local_font_path = resource_manager.get_path(font_path) + font = ImageFont.truetype(local_font_path, 200) + TEXT_COORDS = [ + (525, 1341), + (1589, 1341), + (525, 2889), + (1589, 2889), + ] + + pages = [] + for (image_path, image_bits) in IMAGE_DATA: + requested_bits = int(requested_bits) + if image_bits <= requested_bits: + data = resource_manager.load(os.path.join(BASE_IMAGE_PATH, image_path)) + image = Image.open(data) + if dot_counts: + draw = ImageDraw.Draw(image) + for number in range(image_bits - 4, image_bits): + text = str(pow(2, number)) + text_width, text_height = draw.textsize(text, font=font) + coord_x = TEXT_COORDS[number % 4][0] - (text_width / 2) + coord_y = TEXT_COORDS[number % 4][1] - (text_height / 2) + draw.text( + (coord_x, coord_y), + text, + font=font, + fill="#000" + ) + pages.append({"type": "image", "data": image}) + + if black_back: + black_card = Image.new("1", (IMAGE_SIZE_X, IMAGE_SIZE_Y)) + pages.append({"type": "image", "data": black_card}) + + return pages + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + text for subtitle. (str) + """ + if task["dot_counts"]: + display_numbers_text = "with dot counts" + else: + display_numbers_text = "without dot counts" + + if task["black_back"]: + black_back_text = "with black back" + else: + black_back_text = "without black back" + + text = "{} bits - {} - {} - {}".format( + task["number_bits"], + display_numbers_text, + black_back_text, + task["paper_size"] + ) + return text + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "number_bits": ["4", "8", "12"], + "dot_counts": [True, False], + "black_back": [True, False], + "paper_size": ["a4", "letter"], + } diff --git a/renderservice/render/resources/binary_to_alphabet.py b/renderservice/render/resources/binary_to_alphabet.py new file mode 100644 index 0000000..7776936 --- /dev/null +++ b/renderservice/render/resources/binary_to_alphabet.py @@ -0,0 +1,115 @@ +"""Module for generating Binary to Alphabet resource.""" + +from PIL import Image, ImageDraw, ImageFont + + +def resource(task, resource_manager): + """Create a image for Binary to Alphabet resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + # Retrieve relevant image + worksheet_version = task["worksheet_version"] + if worksheet_version == "student": + image_path = "img/resources/binary-to-alphabet/table.png" + else: + image_path = "img/resources/binary-to-alphabet/table-teacher.png" + data = resource_manager.load(image_path) + image = Image.open(data) + draw = ImageDraw.Draw(image) + + font_size = 30 + font_path = "fonts/PatrickHand-Regular.ttf" + local_font_path = resource_manager.get_path(font_path) + font = ImageFont.truetype(local_font_path, font_size) + + # Draw headings + column_headings = ["Base 10", "Binary", "Letter"] + heading_coord_x = 18 + heading_coord_y = 6 + + i = 0 + while i < 9: # 9 = number of columns + + if i % 3 == 0: + text = str(column_headings[0]) + elif i % 3 == 1: + text = str(column_headings[1]) + else: + text = str(column_headings[2]) + + draw.text( + (heading_coord_x, heading_coord_y), + text, + font=font, + fill="#000" + ) + + heading_coord_x += 113 + + i += 1 + + # Draw numbers + # Column data: (min number, max number), x coord + columns_data = [((0, 9), 58), ((9, 18), 397), ((18, 27), 736)] + + for column_set in columns_data: + start, end = column_set[0] + base_coord_x = column_set[1] + base_coord_y = 75 + + for number in range(start, end): + text = str(number) + text_width, text_height = draw.textsize(text, font=font) + coord_x = base_coord_x - (text_width / 2) + coord_y = base_coord_y - (text_height / 2) + + draw.text( + (coord_x, coord_y), + text, + font=font, + fill="#000" + ) + + base_coord_y += 54 + + image = image.rotate(90, expand=True) + return {"type": "image", "data": image} + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + Text for subtitle. (str) + """ + text = "{} - {}".format( + task["worksheet_version"], + task["paper_size"] + ) + return text + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "worksheet_version": ["student", "teacher"], + "paper_size": ["a4", "letter"] + } diff --git a/renderservice/render/resources/binary_windows.py b/renderservice/render/resources/binary_windows.py new file mode 100644 index 0000000..7a633f9 --- /dev/null +++ b/renderservice/render/resources/binary_windows.py @@ -0,0 +1,198 @@ +"""Module for generating Binary Windows resource.""" + +import os.path +from PIL import Image, ImageDraw, ImageFont + + +def resource(task, resource_manager): + """Create a image for Binary Windows resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + BASE_IMAGE_PATH = "img/resources/binary-windows/" + FONT_PATH = "fonts/PatrickHand-Regular.ttf" + + local_font_path = resource_manager.get_path(FONT_PATH) + font = ImageFont.truetype(local_font_path, 300) + small_font = ImageFont.truetype(local_font_path, 180) + + # Retrieve parameters + number_of_bits = task["number_bits"] + value_type = task["value_type"] + dot_counts = task["dot_counts"] + + pages = [] + page_sets = [("binary-windows-1-to-8.png", 8)] + if number_of_bits == "8": + page_sets.append(("binary-windows-16-to-128.png", 128)) + + for (filename, dot_count_start) in page_sets: + data = resource_manager.load(os.path.join(BASE_IMAGE_PATH, filename)) + image = Image.open(data) + image = add_digit_values(image, resource_manager, value_type, True, 660, 724, 1700, font) + if dot_counts: + image = add_dot_counts(image, dot_count_start, small_font) + image = image.rotate(90, expand=True) + pages.append({"type": "image", "data": image}) + pages.append(back_page(BASE_IMAGE_PATH, resource_manager, font, value_type)) + + return pages + + +def back_page(base_image_path, resource_manager, font, value_type): + """Return a Pillow object of back page of Binary Windows. + + Args: + base_image_path: Base image path for finding images. (str) + font: Pillow ImageFont for writing text. (ImageFont) + value_type: Type of value representation used. (str) + + Returns: + Pillow Image of back page (Image). + """ + data = resource_manager.load(os.path.join(base_image_path, "binary-windows-blank.png")) + image = Image.open(data) + image = add_digit_values(image, resource_manager, value_type, False, 660, 724, 650, font) + image = image.rotate(90, expand=True) + return {"type": "image", "data": image} + + +def add_dot_counts(image, starting_value, font): + """Add dot count text onto image. + + Args: + image: The image to add text to. (Pillow Image) + starting_value: Number on left window. (int) + font: Font used for adding text. (Pillow Font) + + Returns: + Pillow Image with text added. (Pillow Image) + """ + value = starting_value + draw = ImageDraw.Draw(image) + coord_x = 660 + coord_x_increment = 724 + coord_y = 1000 + for i in range(4): + text = str(value) + text_width, text_height = draw.textsize(text, font=font) + text_coord_x = coord_x - (text_width / 2) + text_coord_y = coord_y - (text_height / 2) + draw.text( + (text_coord_x, text_coord_y), + text, + font=font, + fill="#000" + ) + coord_x += coord_x_increment + value = int(value / 2) + return image + + +def add_digit_values(image, resource_manager, value_type, on, x_coord_start, x_coord_increment, base_y_coord, font): + """Add binary values onto image. + + Args: + image: The image to add binary values to. (Pillow Image) + value_type: Either "binary" for 0's and 1's, or "lightbulb" for + lit and unlit lightbulbs. (str) + on: True if binary value is on/lit, otherwise False. (bool) + x_coord_start: X co-ordinate starting value. (int) + x_coord_increment: X co-ordinate increment value. (int) + base_y_coord: Y co-ordinate value. (int) + font: Font used for adding text. (Pillow Font) + + Returns: + Pillow Image with binary values. (Pillow Image) + """ + text_coord_x = x_coord_start + + if value_type == "binary": + if on: + text = "1" + else: + text = "0" + elif value_type == "lightbulb": + if on: + image_file = "img/resources/binary-windows/col_binary_lightbulb.png" + else: + image_file = "img/resources/binary-windows/col_binary_lightbulb_off.png" + data = resource_manager.load(image_file) + lightbulb = Image.open(data) + (width, height) = lightbulb.size + SCALE_FACTOR = 0.6 + lightbulb = lightbulb.resize((int(width * SCALE_FACTOR), int(height * SCALE_FACTOR))) + (width, height) = lightbulb.size + lightbulb_width = int(width / 2) + lightbulb_height = int(height / 2) + if not on: + lightbulb = lightbulb.rotate(180) + + for i in range(4): + draw = ImageDraw.Draw(image) + if value_type == "binary": + + text_width, text_height = draw.textsize(text, font=font) + coord_x = text_coord_x - (text_width / 2) + coord_y = base_y_coord - (text_height / 2) + draw.text( + (coord_x, coord_y), + text, + font=font, + fill="#000" + ) + elif value_type == "lightbulb": + coords = (text_coord_x - lightbulb_width, base_y_coord - lightbulb_height + 75) + image.paste(lightbulb, box=coords, mask=lightbulb) + text_coord_x += x_coord_increment + return image + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + text for subtitle. (str) + """ + number_of_bits = task["number_bits"] + value_type = task["value_type"] + dot_counts = task["dot_counts"] + if dot_counts: + count_text = "with dot counts" + else: + count_text = "without dot counts" + + TEMPLATE = "{num_bits} bits - {value} - {counts}" + text = TEMPLATE.format( + num_bits=number_of_bits, + value=value_type, + counts=count_text + ) + return text + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "number_bits": ["4", "8"], + "value_type": ["binary", "lightbulb"], + "dot_counts": [True, False], + "paper_size": ["a4", "letter"], + } diff --git a/renderservice/render/resources/grid.py b/renderservice/render/resources/grid.py new file mode 100644 index 0000000..6ea243b --- /dev/null +++ b/renderservice/render/resources/grid.py @@ -0,0 +1,61 @@ +"""Module for generating Grid resource.""" + +from PIL import Image, ImageDraw + + +def resource(task, resource_manager): + """Create a image for Grid resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + GRID_COLUMNS = 8 + GRID_ROWS = 8 + BOX_SIZE = 500 + IMAGE_SIZE_X = BOX_SIZE * GRID_COLUMNS + IMAGE_SIZE_Y = BOX_SIZE * GRID_ROWS + LINE_COLOUR = "#000000" + LINE_WIDTH = 3 + + page = Image.new("RGB", (IMAGE_SIZE_X, IMAGE_SIZE_Y), "#fff") + draw = ImageDraw.Draw(page) + for x_coord in range(0, IMAGE_SIZE_X, BOX_SIZE): + draw.line([(x_coord, 0), (x_coord, IMAGE_SIZE_Y)], fill=LINE_COLOUR, width=LINE_WIDTH) + draw.line([(IMAGE_SIZE_X - 1, 0), (IMAGE_SIZE_X - 1, IMAGE_SIZE_Y)], fill=LINE_COLOUR, width=LINE_WIDTH) + for y_coord in range(0, IMAGE_SIZE_Y, BOX_SIZE): + draw.line([(0, y_coord), (IMAGE_SIZE_X, y_coord)], fill=LINE_COLOUR, width=LINE_WIDTH) + draw.line([(0, IMAGE_SIZE_Y - 1), (IMAGE_SIZE_X, IMAGE_SIZE_Y - 1)], fill=LINE_COLOUR, width=LINE_WIDTH) + + return {"type": "image", "data": page} + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + text for subtitle. (str) + """ + return task["paper_size"] + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "paper_size": ["a4", "letter"], + } diff --git a/renderservice/render/resources/job_badges.py b/renderservice/render/resources/job_badges.py new file mode 100644 index 0000000..8814b16 --- /dev/null +++ b/renderservice/render/resources/job_badges.py @@ -0,0 +1,48 @@ +"""Module for generating Job Badges resource.""" + +from PIL import Image, ImageDraw + + +def resource(task, resource_manager): + """Create a image for Job Badges resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. (dict) + """ + image_path = "img/resources/job-badges/job-badges.png" + data = resource_manager.load(image_path) + image = Image.open(data) + ImageDraw.Draw(image) + return {"type": "image", "data": image} + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. + + Returns: + Text for subtitle. (str) + """ + return task["paper_size"] + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options (dict). + """ + return { + "paper_size": ["a4", "letter"] + } diff --git a/renderservice/render/resources/left_right_cards.py b/renderservice/render/resources/left_right_cards.py new file mode 100644 index 0000000..c9de23e --- /dev/null +++ b/renderservice/render/resources/left_right_cards.py @@ -0,0 +1,48 @@ +"""Module for generating Left and Right Cards resource.""" + +from PIL import Image, ImageDraw + + +def resource(task, resource_manager): + """Create a image for Left and Right Cards resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + image_path = "img/resources/left-right-cards/left-right-cards.png" + data = resource_manager.load(image_path) + image = Image.open(data) + ImageDraw.Draw(image) + return {"type": "image", "data": image} + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + Text for subtitle. (str) + """ + return task["paper_size"] + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "paper_size": ["a4", "letter"] + } diff --git a/renderservice/render/resources/modulo_clock.py b/renderservice/render/resources/modulo_clock.py new file mode 100644 index 0000000..45b795e --- /dev/null +++ b/renderservice/render/resources/modulo_clock.py @@ -0,0 +1,87 @@ +"""Module for generating Module Clock resource.""" + +from math import pi, sin, cos +from PIL import Image, ImageDraw, ImageFont + + +def resource(task, resource_manager): + """Create a image for Modulo Clock resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + image_path = "img/resources/modulo-clock/modulo-clock-{}.png" + + modulo_number = int(task["modulo_number"]) + data = resource_manager.load(image_path.format(modulo_number)) + image = Image.open(data) + draw = ImageDraw.Draw(image) + + font_size = 150 + font_path = "fonts/PatrickHand-Regular.ttf" + local_font_path = resource_manager.get_path(font_path) + font = ImageFont.truetype(local_font_path, font_size) + + radius = 750 + x_center = image.width / 2 + y_center = image.height / 2 + # Count from the '12 oclock position' + start_angle = pi * 1.5 + angle_between_numbers = (2 * pi) / modulo_number + for number in range(0, modulo_number): + text = str(number) + angle = start_angle + (angle_between_numbers * number) + x_coord = radius * cos(angle) + x_center + y_coord = radius * sin(angle) + y_center + text_width, text_height = draw.textsize(text, font=font) + text_coord_x = x_coord - (text_width / 2) + text_coord_y = y_coord - (text_height / 2) + draw.text( + (text_coord_x, text_coord_y), + text, + font=font, + fill="#000" + ) + + return {"type": "image", "data": image} + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + Text for subtitle. (str) + """ + modulo_number = task["modulo_number"] + if modulo_number == "1": + modulo_text = "blank" + else: + modulo_text = modulo_number + return "{} - {}".format( + modulo_text, + task["paper_size"] + ) + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "modulo_number": ["1", "2", "10"], + "paper_size": ["a4", "letter"] + } diff --git a/renderservice/render/resources/parity_cards.py b/renderservice/render/resources/parity_cards.py new file mode 100644 index 0000000..95065de --- /dev/null +++ b/renderservice/render/resources/parity_cards.py @@ -0,0 +1,91 @@ +"""Module for generating Parity Cards resource.""" + +from PIL import Image, ImageDraw + + +def resource(task, resource_manager): + """Create a image for Parity Cards resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + CARDS_COLUMNS = 4 + CARDS_ROWS = 5 + CARD_SIZE = 500 + IMAGE_SIZE_X = CARD_SIZE * CARDS_COLUMNS + IMAGE_SIZE_Y = CARD_SIZE * CARDS_ROWS + LINE_COLOUR = "#000000" + LINE_WIDTH = 3 + + front_page = Image.new("RGB", (IMAGE_SIZE_X, IMAGE_SIZE_Y), "#fff") + draw = ImageDraw.Draw(front_page) + for x_coord in range(0, IMAGE_SIZE_X, CARD_SIZE): + draw.line([(x_coord, 0), (x_coord, IMAGE_SIZE_Y)], fill=LINE_COLOUR, width=LINE_WIDTH) + draw.line([(IMAGE_SIZE_X - 1, 0), (IMAGE_SIZE_X - 1, IMAGE_SIZE_Y)], fill=LINE_COLOUR, width=LINE_WIDTH) + for y_coord in range(0, IMAGE_SIZE_Y, CARD_SIZE): + draw.line([(0, y_coord), (IMAGE_SIZE_X, y_coord)], fill=LINE_COLOUR, width=LINE_WIDTH) + draw.line([(0, IMAGE_SIZE_Y - 1), (IMAGE_SIZE_X, IMAGE_SIZE_Y - 1)], fill=LINE_COLOUR, width=LINE_WIDTH) + + # Retrieve parameters + back_colour = task["back_colour"] + + if back_colour == "black": + back_colour_hex = "#000000" + elif back_colour == "blue": + back_colour_hex = "#3366ff" + elif back_colour == "green": + back_colour_hex = "#279f2d" + elif back_colour == "purple": + back_colour_hex = "#6e3896" + else: + back_colour_hex = "#cc0423" + + pages = [ + { + "type": "image", + "data": front_page + }, + { + "type": "image", + "data": Image.new("RGB", (IMAGE_SIZE_X, IMAGE_SIZE_Y), back_colour_hex) + } + ] + + return pages + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document options. (dict) + + Returns: + text for subtitle. (str) + """ + text = "{} back - {}".format( + task["back_colour"], + task["paper_size"] + ) + return text + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "back_colour": ["black", "blue", "green", "purple", "red"], + "paper_size": ["a4", "letter"], + } diff --git a/renderservice/render/resources/piano_keys.py b/renderservice/render/resources/piano_keys.py new file mode 100644 index 0000000..e6b9eff --- /dev/null +++ b/renderservice/render/resources/piano_keys.py @@ -0,0 +1,126 @@ +"""Module for generating Piano Keys resource.""" + +from PIL import Image, ImageDraw +from render.resources.utils import bool_to_yes_no_or_pass_thru + + +def resource(task, resource_manager): + """Create a image for Piano Keys resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + KEY_DATA = { + "A": { + "colour": "hsl(356, 95%, 85%)", + "areas": [ + ((1002, 21), (1005, 711), (1200, 716), (1193, 15)), + ((2392, 15), (2356, 720), (2541, 720), (2552, 17)), + ], + }, + "B": { + "colour": "hsl(21, 90%, 85%)", + "areas": [ + ((1193, 15), (1200, 716), (1369, 719), (1400, 15)), + ((2552, 17), (2541, 720), (2713, 720), (2756, 20)), + ], + }, + "C": { + "colour": "hsl(52, 100%, 85%)", + "areas": [ + ((15, 15), (51, 711), (255, 715), (186, 21)), + ((1395, 24), (1371, 720), (1566, 717), (1590, 18)), + ], + }, + "D": { + "colour": "hsl(140, 87%, 85%)", + "areas": [ + ((186, 21), (255, 715), (408, 714), (390, 15)), + ((1590, 18), (1566, 717), (1760, 718), (1794, 12)), + ], + }, + "E": { + "colour": "hsl(205, 85%, 85%)", + "areas": [ + ((390, 15), (408, 714), (603, 717), (585, 12)), + ((1794, 12), (1760, 718), (1979, 720), (2004, 19)), + ], + }, + "F": { + "colour": "hsl(293, 45%, 85%)", + "areas": [ + ((585, 12), (603, 717), (828, 720), (717, 15)), + ((2004, 19), (1979, 720), (2163, 720), (2192, 15)), + ], + }, + "G": { + "colour": "hsl(238, 51%, 85%)", + "areas": [ + ((717, 15), (828, 720), (1005, 711), (1002, 21)), + ((2192, 15), (2163, 720), (2356, 720), (2392, 15)), + ], + }, + } + + highlight = task["highlight"] + image_path = "img/resources/piano-keys/keyboard.png" + data = resource_manager.load(image_path) + image = Image.open(data) + page = Image.new("RGB", image.size, "#FFF") + + if highlight: + highlight_key_areas(page, KEY_DATA.get(highlight)) + + # Add piano keys overlay + page.paste(image, mask=image) + + page = page.rotate(90, expand=True) + return {"type": "image", "data": page} + + +def highlight_key_areas(image, key_data): + """Highlights the page of keys. + + Args: + image: PillowImage of page. (Pillow Image) + key_data: Dictionary of highlight colour and areas. (dict) + """ + draw = ImageDraw.Draw(image) + for area in key_data["areas"]: + draw.polygon(area, fill=key_data["colour"]) + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + text for subtitle. (str) + """ + return "{} highlight - {}".format( + bool_to_yes_no_or_pass_thru(task["highlight"]), + task["paper_size"] + ) + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "highlight": [False, "A", "B", "C", "D", "E", "F", "G"], + "paper_size": ["a4", "letter"], + } diff --git a/renderservice/render/resources/searching_cards.py b/renderservice/render/resources/searching_cards.py new file mode 100644 index 0000000..f50ceaa --- /dev/null +++ b/renderservice/render/resources/searching_cards.py @@ -0,0 +1,160 @@ +"""Module for generating Searching Cards resource.""" + +from random import sample, shuffle +from math import ceil +from PIL import Image, ImageDraw, ImageFont +from yattag import Doc + + +def resource(task, resource_manager): + """Create a copy of the Searching Cards resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + pages = [] + IMAGE_PATH = "img/resources/searching-cards/{}-cards-{}.png" + X_BASE_COORD = 1803 + X_COORD_DECREMENT = 516 + Y_COORD = 240 + FONT_PATH = "fonts/PatrickHand-Regular.ttf" + local_font_path = resource_manager.get_path(FONT_PATH) + font = ImageFont.truetype(local_font_path, 200) + + number_cards = int(task["number_cards"]) + max_number = task["max_number"] + help_sheet = task["help_sheet"] + + if max_number == "cards": + numbers = list(range(1, number_cards + 1)) + shuffle(numbers) + range_text = "1 to {}".format(number_cards) + elif max_number != "blank": + numbers = sample(range(1, int(max_number) + 1), number_cards) + range_text = "1 to {}".format(max_number) + else: + numbers = [] + range_text = "Add list of numbers below:" + + if help_sheet: + pages.append({"type": "html", "data": create_help_sheet(numbers, range_text)}) + + number_of_pages = range(ceil(number_cards / 4)) + for page in number_of_pages: + if page == number_of_pages[-1]: + image_path = IMAGE_PATH.format(3, 1) + else: + image_path = IMAGE_PATH.format(4, page + 1) + + data = resource_manager.load(image_path) + image = Image.open(data) + + if max_number != "blank": + draw = ImageDraw.Draw(image) + page_numbers = numbers[:4] + numbers = numbers[4:] + coord_x = X_BASE_COORD + for number in page_numbers: + text = str(number) + text_width, text_height = draw.textsize(text, font=font) + draw.text( + (coord_x - (text_width / 2), Y_COORD - (text_height / 2)), + text, + font=font, + fill="#000" + ) + coord_x -= X_COORD_DECREMENT + + image = image.rotate(90, expand=True) + pages.append({"type": "image", "data": image}) + return pages + + +def create_help_sheet(numbers, range_text): + """Create helper sheet for resource. + + Args: + numbers: Numbers used for activity. (list) + range_text: String describing range of numbers. (str) + + Returns: + Pillow image object. (Image) + """ + doc, tag, text, line = Doc().ttl() + with tag("div"): + with tag("h1"): + text("Helper page for binary search activity") + with tag("p"): + text( + "Use this sheet to circle the number you are asking your class ", + "to look for when you are demonstrating how the binary search ", + "works. This allows you to demonstrate the maximum number of ", + "searches it would take. When students are playing the treasure ", + "hunt game, they can choose any number. Avoid those that are in ", + "red as they are key binary search positions (avoiding them is a ", + "good thing to do for demonstrations, but in practice students, ", + "or computers, won’t intentionally avoid these)." + ) + with tag("h2"): + text("Sorted numbers") + with tag("ul", klass="list-unstyled"): + numbers.sort() + red_number_jump = (len(numbers) + 1) // 4 + for (index, number) in enumerate(numbers): + if (index + 1) % red_number_jump == 0: + line("li", number, klass="text-danger") + else: + line("li", number) + return doc.getvalue() + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + text for subtitle. (str) + """ + number_cards = task["number_cards"] + max_number = task["max_number"] + help_sheet = task["help_sheet"] + paper_size = task["paper_size"] + + if max_number == "blank": + range_text = "blank" + elif max_number == "cards": + range_text = "0 to {}".format(number_cards) + else: + range_text = "0 to {}".format(max_number) + + if help_sheet: + help_text = "with helper sheet" + else: + help_text = "without helper sheet" + + return "{} cards - {} - {} - {}".format(number_cards, range_text, help_text, paper_size) + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "number_cards": ["15", "31"], + "max_number": ["cards", "99", "999", "blank"], + "help_sheet": [True, False], + "paper_size": ["a4", "letter"], + } diff --git a/renderservice/render/resources/sorting_network.py b/renderservice/render/resources/sorting_network.py new file mode 100644 index 0000000..88dd37f --- /dev/null +++ b/renderservice/render/resources/sorting_network.py @@ -0,0 +1,111 @@ +"""Module for generating Sorting Network resource.""" + +from PIL import Image, ImageDraw, ImageFont +from random import sample + + +def resource(task, resource_manager): + """Create a image for Sorting Network resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + image_path = "img/resources/resource-sorting-network-colour.png" + data = resource_manager.load(image_path) + image = Image.open(data) + draw = ImageDraw.Draw(image) + + font_path = "fonts/PatrickHand-Regular.ttf" + local_font_path = resource_manager.get_path(font_path) + + (range_min, range_max, font_size) = number_range(task) + + # Add numbers to text if needed + prefilled_values = task["prefilled_values"] + if prefilled_values != "blank": + font = ImageFont.truetype(local_font_path, font_size) + numbers = sample(range(range_min, range_max), 6) + base_coord_x = 70 + base_coord_y = 2560 + coord_x_increment = 204 + for number in numbers: + text = str(number) + text_width, text_height = draw.textsize(text, font=font) + coord_x = base_coord_x - (text_width / 2) + coord_y = base_coord_y - (text_height / 2) + draw.text( + (coord_x, coord_y), + text, + font=font, + fill="#000" + ) + base_coord_x += coord_x_increment + + return {"type": "image", "data": image} + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + text for subtitle. (str) + """ + prefilled_values = task["prefilled_values"] + if prefilled_values == "blank": + range_text = "blank" + else: + SUBTITLE_TEMPLATE = "{} to {}" + range_min, range_max, font_size = number_range(task) + range_text = SUBTITLE_TEMPLATE.format(range_min, range_max - 1) + return "{} - {}".format(range_text, task["paper_size"]) + + +def number_range(task): + """Return number range tuple for resource. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + Tuple of range_min, range_max, font_size. (tuple) + """ + prefilled_values = task["prefilled_values"] + range_min = 0 + range_max = 0 + font_size = 150 + if prefilled_values == "easy": + range_min = 1 + range_max = 10 + elif prefilled_values == "medium": + range_min = 10 + range_max = 100 + font_size = 120 + elif prefilled_values == "hard": + range_min = 100 + range_max = 1000 + font_size = 90 + return (range_min, range_max, font_size) + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "prefilled_values": ["blank", "easy", "medium", "hard"], + "paper_size": ["a4", "letter"], + } diff --git a/renderservice/render/resources/sorting_network_cards.py b/renderservice/render/resources/sorting_network_cards.py new file mode 100644 index 0000000..a18709b --- /dev/null +++ b/renderservice/render/resources/sorting_network_cards.py @@ -0,0 +1,187 @@ +"""Module for generating Sorting Network Cards resource.""" + +import os +from random import sample +from PIL import Image, ImageDraw, ImageFont + + +def resource(task, resource_manager): + """Create a image for Sorting Network Cards resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + IMAGE_SIZE_X = 2000 + IMAGE_SIZE_Y = 3000 + LINE_COLOUR = "#000000" + LINE_WIDTH = 3 + + font_path = "fonts/PatrickHand-Regular.ttf" + local_font_path = resource_manager.get_path(font_path) + font_size = 300 + + # Retrieve parameters + card_type = task["type"] + + # Create card outlines + card_outlines = Image.new("RGB", (IMAGE_SIZE_X, IMAGE_SIZE_Y), "#fff") + draw = ImageDraw.Draw(card_outlines) + for x_coord in range(0, IMAGE_SIZE_X, IMAGE_SIZE_X - LINE_WIDTH): + draw.line([(x_coord, 0), (x_coord, IMAGE_SIZE_Y)], fill=LINE_COLOUR, width=LINE_WIDTH) + for y_coord in range(0, IMAGE_SIZE_Y, int(IMAGE_SIZE_Y / 2 - LINE_WIDTH)): + draw.line([(0, y_coord), (IMAGE_SIZE_X, y_coord)], fill=LINE_COLOUR, width=LINE_WIDTH) + + # Prepare text data + card_data_type = "text" + if card_type == "small_numbers": + font_size = 800 + text = ["1", "2", "3", "4", "5", "6"] + elif card_type == "large_numbers": + font_size = 500 + text = [] + numbers = sample(range(1700000, 2100000), 6) + for number in numbers: + text.append("{:,}".format(number)) + elif card_type == "fractions": + font_size = 900 + font_path = "fonts/NotoSans-Regular.ttf" + local_font_path = resource_manager.get_path(font_path) + text = [u"\u00bd", u"\u2153", u"\u2154", u"\u215c", u"\u00be", u"\u215d"] + elif card_type == "maori_numbers": + font_size = 300 + text = [ + "tahi", "rua", "toru", "whā", "rima", "ono", "whitu", "waru", + "iwa", "tekau", "tekau mā tahi", "tekau mā waru", "tekau mā toru", + "tekau mā whā", "rua tekau", "rua tekau mā ono" + ] + elif card_type == "words": + font_size = 500 + text = ["crocodile", "crochet", "kiwi", "weka", "kiwi", "kiwano"] + elif card_type == "letters": + font_size = 800 + text = ["L", "O", "N", "K", "E", "D", "S", "P", "G", "B", "I", "Y"] + elif card_type == "maori_colours": + font_size = 500 + text = [ + "whero", "kākāriki", "kiwikiwi", "karaka", + "kōwhai", "pango", "māwhero", "mā" + ] + elif card_type == "riding_hood": + card_data_type = "image" + images = [ + "little-red-riding-hood-1.png", + "little-red-riding-hood-2.png", + "little-red-riding-hood-3.png", + "little-red-riding-hood-4.png", + "little-red-riding-hood-5.png", + "little-red-riding-hood-6.png", + ] + elif card_type == "butterfly": + card_data_type = "image" + images = [ + "butterfly-story-leaf.png", + "butterfly-story-baby-caterpillar.png", + "butterfly-story-caterpillar.png", + "butterfly-story-young-pupa.png", + "butterfly-story-mature-pupa.png", + "butterfly-story-butterfly.png", + ] + + card_centers = [ + (IMAGE_SIZE_X / 2, IMAGE_SIZE_Y / 4), + (IMAGE_SIZE_X / 2, (IMAGE_SIZE_Y / 4) * 3), + ] + + # Add text to cards + pages = [] + if card_data_type == "image": + IMAGE_PADDING = 20 + MAX_IMAGE_X = IMAGE_SIZE_X - IMAGE_PADDING * 2 + MAX_IMAGE_Y = IMAGE_SIZE_Y / 2 - IMAGE_PADDING * 2 + BASE_PATH = "img/resources/sorting-network-cards/" + for (image_number, filename) in enumerate(images): + data = resource_manager.load(os.path.join(BASE_PATH, filename)) + image = Image.open(data) + (width, height) = image.size + if height > MAX_IMAGE_Y or width > MAX_IMAGE_X: + height_ratio = MAX_IMAGE_Y / height + width_ratio = MAX_IMAGE_X / width + ratio = min(height_ratio, width_ratio) + width *= ratio + height *= ratio + image = image.resize((int(width), int(height)), Image.ANTIALIAS) + if image_number % 2 == 0: + page = card_outlines.copy() + draw = ImageDraw.Draw(page) + (x, y) = card_centers[0] + else: + (x, y) = card_centers[1] + coords = (int(x - (width / 2)), int(y - (height / 2))) + page.paste(image, box=coords, mask=image) + # If image on second card but not last page + if image_number % 2 == 1 and image_number != len(images) - 1: + pages.append({"type": "image", "data": page}) + else: + font = ImageFont.truetype(local_font_path, font_size) + for (text_number, text_string) in enumerate(text): + if text_number % 2 == 0: + page = card_outlines.copy() + draw = ImageDraw.Draw(page) + (x, y) = card_centers[0] + else: + (x, y) = card_centers[1] + + text_width, text_height = draw.textsize(text_string, font=font) + coord_x = x - (text_width / 2) + coord_y = y - (text_height / 1.5) + draw.text( + (coord_x, coord_y), + text_string, + font=font, + fill="#000" + ) + # If text on second card but not last page + if text_number % 2 == 1 and text_number != len(text) - 1: + pages.append({"type": "image", "data": page}) + + pages.append({"type": "image", "data": page}) + return pages + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + text for subtitle. (str) + """ + return "{} - {}".format( + task["type"].replace("_", " "), + task["paper_size"] + ) + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "type": [ + "small_numbers", "large_numbers", "fractions", "maori_numbers", + "words", "letters", "maori_colours", "riding_hood", "butterfly" + ], + "paper_size": ["a4", "letter"], + } diff --git a/renderservice/render/resources/train_stations.py b/renderservice/render/resources/train_stations.py new file mode 100644 index 0000000..30e5d06 --- /dev/null +++ b/renderservice/render/resources/train_stations.py @@ -0,0 +1,53 @@ +"""Module for generating Train Stations resource.""" + +from PIL import Image + + +def resource(task, resource_manager): + """Create a image for Train Stations resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + image_path = "img/resources/train-stations/train-stations-tracks-{}.png" + track_type = task["tracks"] + data = resource_manager.load(image_path.format(track_type)) + image = Image.open(data) + image = image.rotate(90, expand=True) + return {"type": "image", "data": image} + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + Text for subtitle. (str) + """ + return "{} tracks - {}".format( + task["tracks"], + task["paper_size"] + ) + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "tracks": ["circular", "twisted"], + "paper_size": ["a4", "letter"] + } diff --git a/renderservice/render/resources/treasure_hunt.py b/renderservice/render/resources/treasure_hunt.py new file mode 100644 index 0000000..f6ccec2 --- /dev/null +++ b/renderservice/render/resources/treasure_hunt.py @@ -0,0 +1,158 @@ +"""Module for generating Treasure Hunt resource.""" + +from PIL import Image, ImageDraw, ImageFont +from random import sample + + +def resource(task, resource_manager): + """Create a image for Treasure Hunt resource. + + Args: + task: Dicitionary of requested document options. (dict) + resource_manager: File loader for external resources. (FileManager) + + Returns: + A dictionary or list of dictionaries for each resource page. + """ + pages = [] + IMAGE_PATH = "img/resources/treasure-hunt/{}.png" + FONT_PATH = "fonts/PatrickHand-Regular.ttf" + local_font_path = resource_manager.get_path(FONT_PATH) + + # Add numbers to image if required + prefilled_values = task["prefilled_values"] + number_order = task["number_order"] + instructions = task["instructions"] + art_style = task["art"] + + if instructions: + data = resource_manager.load(IMAGE_PATH.format("instructions")) + image = Image.open(data) + ImageDraw.Draw(image) + pages.append({"type": "image", "data": image}) + + data = resource_manager.load(IMAGE_PATH.format(art_style)) + image = Image.open(data) + draw = ImageDraw.Draw(image) + + # Add numbers to image if required + if prefilled_values != "blank": + (range_min, range_max, font_size) = number_range(task) + font = ImageFont.truetype(local_font_path, font_size) + + total_numbers = 26 + numbers = sample(range(range_min, range_max), total_numbers) + if number_order == "sorted": + numbers.sort() + + base_coord_y = 506 + coord_y_increment = 199 + base_coords_x = [390, 700] + for i in range(0, total_numbers): + text = str(numbers[i]) + text_width, text_height = draw.textsize(text, font=font) + + coord_x = base_coords_x[i % 2] - (text_width / 2) + coord_y = base_coord_y - (text_height / 2) + if i % 2 == 1: + coord_y -= 10 + base_coord_y += coord_y_increment + draw.text( + (coord_x, coord_y), + text, + font=font, + fill="#000" + ) + + text = "{} - {} to {}".format(number_order.title(), range_min, range_max - 1) + font = ImageFont.truetype(local_font_path, 75) + text_width, text_height = draw.textsize(text, font=font) + coord_x = 1220 - (text_width / 2) + coord_y = 520 - (text_height / 2) + draw.text( + (coord_x, coord_y), + text, + font=font, + fill="#000", + ) + pages.append({"type": "image", "data": image}) + + return pages + + +def subtitle(task): + """Return the subtitle string of the resource. + + Used after the resource name in the filename, and + also on the resource image. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + text for subtitle. (str) + """ + prefilled_values = task["prefilled_values"] + art_style = task["art"] + instructions = task["instructions"] + paper_size = task["paper_size"] + + if prefilled_values == "blank": + range_text = "blank" + else: + SUBTITLE_TEMPLATE = "{} - {} to {}" + number_order_text = task["number_order"].title() + range_min, range_max, font_size = number_range(task) + range_text = SUBTITLE_TEMPLATE.format(number_order_text, range_min, range_max - 1) + + if art_style == "colour": + art_style_text = "full colour" + else: + art_style_text = "black and white" + + if instructions: + instructions_text = "with instructions" + else: + instructions_text = "without instructions" + + return "{} - {} - {} - {}".format(range_text, art_style_text, instructions_text, paper_size) + + +def number_range(task): + """Return number range tuple for resource. + + Args: + task: Dicitionary of requested document. (dict) + + Returns: + Tuple of range_min, range_max, font_size. (tuple) + """ + prefilled_values = task["prefilled_values"] + range_min = 0 + if prefilled_values == "easy": + range_max = 100 + font_size = 55 + elif prefilled_values == "medium": + range_max = 1000 + font_size = 50 + elif prefilled_values == "hard": + range_max = 10000 + font_size = 45 + return (range_min, range_max, font_size) + + +def valid_options(): + """Provide dictionary of all valid parameters. + + This excludes the header text parameter. + + Returns: + All valid options. (dict) + """ + return { + "prefilled_values": ["blank", "easy", "medium", "hard"], + "number_order": ["sorted", "unsorted"], + "instructions": [True, False], + "art": ["colour", "bw"], + "paper_size": ["a4", "letter"], + } diff --git a/renderservice/render/resources/utils.py b/renderservice/render/resources/utils.py new file mode 100644 index 0000000..e3514e0 --- /dev/null +++ b/renderservice/render/resources/utils.py @@ -0,0 +1,39 @@ +"""Common utility functions for resources.""" + + +def bool_to_yes_no(boolean): + """Convert value to yes or no. + + Args: + boolean: Value to check. (bool) + + Returns: + "yes" if boolean is True, "no" if False. (string) + + Raises: + ValueError if value isn't "yes" or "no". + """ + if type(boolean) is bool and boolean: + return "yes" + elif type(boolean) is bool: + return "no" + raise ValueError("Expected True or False.") + + +def bool_to_yes_no_or_pass_thru(value): + """Convert value if boolean to yes or no. + + Args: + value: Value to check. (string) + + Returns: + "yes" if boolean is True, "no" if False, + otherwise the value is returned. (bool) + + Raises: + ValueError if value isn't "yes" or "no". + """ + try: + return bool_to_yes_no(value) + except ValueError: + return value diff --git a/renderservice/render/tests/BaseTest.py b/renderservice/render/tests/BaseTest.py new file mode 100644 index 0000000..9d0a4da --- /dev/null +++ b/renderservice/render/tests/BaseTest.py @@ -0,0 +1,17 @@ +"""The BaseTest case for tests to inheirit from for render tests.""" +from unittest import TestCase + + +class BaseTest(TestCase): + """A base test class for individual test classes.""" + + def __init__(self, *args, **kwargs): + """Create a Base Test. + + Create class inheiriting from TestCase, while also storing + the path to test files and the maxiumum difference to display on + test failures. + """ + super(BaseTest, self).__init__(*args, **kwargs) + self.test_file_path = "tests/assets/{test_type}/{filename}" + self.maxDiff = None diff --git a/renderservice/render/tests/__init__.py b/renderservice/render/tests/__init__.py new file mode 100644 index 0000000..35f8f00 --- /dev/null +++ b/renderservice/render/tests/__init__.py @@ -0,0 +1 @@ +"""Testing module for the render service.""" diff --git a/renderservice/render/tests/daemon/test_daemon_utils.py b/renderservice/render/tests/daemon/test_daemon_utils.py new file mode 100644 index 0000000..e1dfee8 --- /dev/null +++ b/renderservice/render/tests/daemon/test_daemon_utils.py @@ -0,0 +1,83 @@ +"""Test the file manager for reading of and writing to files.""" +import logging +import subprocess +from render.tests.BaseTest import BaseTest +from render.daemon.utils import check_pid, get_active_daemon_details + +logger = logging.getLogger(__name__) + + +class DaemonUtilsTest(BaseTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @classmethod + def setUpClass(cls): + cls.daemon_number = 9999 # probably not more than 9999 daemons + args = ["/docker_venv/bin/python", "-m", "render.daemon", + "--daemon", str(cls.daemon_number), + "start"] + p = subprocess.Popen(args, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + + cls.returncode = None + for _ in range(20): + try: + cls.returncode = p.wait(1) + except subprocess.TimeoutExpired: + pass + + if cls.returncode is None: + p.terminate() + p.wait() + + @classmethod + def tearDownClass(cls): + args = ["/docker_venv/bin/python", "-m", "render.daemon", + "--daemon", str(cls.daemon_number), + "stop"] + p = subprocess.Popen(args, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + + returncode = None + for _ in range(20): + try: + returncode = p.wait(1) + except subprocess.TimeoutExpired: + pass + + if returncode != 0: + logger.warning("Daemon {} failed to be stopped, please do so manually.") + + if returncode is None: + p.terminate() + p.wait() + + def test_get_active_daemon_details(self): + self.assertEqual(self.returncode, 0) + + details = get_active_daemon_details("render") + self.assertGreaterEqual(len(details), 1) + + numbers = {daemon.number for daemon in details} + self.assertTrue(self.daemon_number in numbers) + + def test_check_pid(self): + self.assertEqual(self.returncode, 0) + + details = get_active_daemon_details("render") + self.assertGreaterEqual(len(details), 1) + + pid = 0 + for daemon in details: + if daemon.number == self.daemon_number: + pid = daemon.pid + self.assertNotEqual(pid, 0) + + self.assertTrue(check_pid(pid)) + self.assertFalse(check_pid(pid + 8192)) # VM should probably not have more that 8000 processes running diff --git a/renderservice/render/tests/daemon/test_file_manager.py b/renderservice/render/tests/daemon/test_file_manager.py new file mode 100644 index 0000000..9960b8b --- /dev/null +++ b/renderservice/render/tests/daemon/test_file_manager.py @@ -0,0 +1,76 @@ +"""Test the file manager for reading of and writing to files.""" +import os +import shutil +from render.tests.BaseTest import BaseTest +from render.daemon.FileManager import FileManager + + +class FileManagerTest(BaseTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @classmethod + def setUpClass(cls): + cwd = os.getcwd() + cls.folderpath_one = os.path.join(cwd, "test_assets") + cls.folderpath_two = os.path.join(cwd, "other_test_assets") + os.makedirs(cls.folderpath_one, exist_ok=False) + os.makedirs(cls.folderpath_two, exist_ok=False) + + cls.testfile_name = "test.txt" + cls.testfile_contents = "This is a testing file." + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.folderpath_one, ignore_errors=False) + shutil.rmtree(cls.folderpath_two, ignore_errors=False) + + def setUp(self): + filepath = os.path.join(self.folderpath_one, self.testfile_name) + with open(filepath, "wb") as f: + f.write(self.testfile_contents.encode("ascii")) + + def test_load_file(self): + file_manager = FileManager(self.folderpath_one) + data = file_manager.load(self.testfile_name) + string = data.read().decode("ascii") + self.assertEqual(string, self.testfile_contents) + + def test_load_file_multiple_directories(self): + file_manager = FileManager(self.folderpath_two, self.folderpath_one) + data = file_manager.load(self.testfile_name) + string = data.read().decode("ascii") + self.assertEqual(string, self.testfile_contents) + + def test_save_file_with_directory(self): + file_manager = FileManager(self.folderpath_one, self.folderpath_two, save_directory=self.folderpath_two) + + filename = "save_with_directory.txt" + file_content = "Hello world!" + file_manager.save(filename, file_content.encode("ascii")) + + data = file_manager.load(filename) + string = data.read().decode("ascii") + self.assertEqual(string, file_content) + + def test_save_file_without_directory(self): + file_manager = FileManager(self.folderpath_one, self.folderpath_two) + + filename = "save_without_directory.txt" + file_content = "Hello world!" + file_manager.save(filename, file_content.encode("ascii")) + + data = file_manager.load(filename) + string = data.read().decode("ascii") + self.assertEqual(string, file_content) + + def test_save_file_without_directory_overwrite(self): + file_manager = FileManager(self.folderpath_two, self.folderpath_one) + + file_content = "Hello world!" + file_manager.save(self.testfile_name, file_content.encode("ascii")) + + data = file_manager.load(self.testfile_name) + string = data.read().decode("ascii") + self.assertEqual(string, file_content) diff --git a/renderservice/render/tests/daemon/test_queue_handler.py b/renderservice/render/tests/daemon/test_queue_handler.py new file mode 100644 index 0000000..b941804 --- /dev/null +++ b/renderservice/render/tests/daemon/test_queue_handler.py @@ -0,0 +1,140 @@ +"""Test the file manager for reading of and writing to files.""" +import os +import time +from render.tests.BaseTest import BaseTest +from render.daemon.QueueHandler import QueueHandler + +DISCOVERY_URL = os.getenv("API_DISCOVERY_URL", None) + + +class QueueHandlerTest(BaseTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.queue = QueueHandler(project_name="test_project", + taskqueue_name="test_queue", + discovery_url=DISCOVERY_URL) + self.clear_queue() + self.assertEqual(len(self.queue), 0) + + def clear_queue(self): + while len(self.queue) > 0: + tasks = self.queue.list_tasks() + for task in tasks: + self.queue.delete_task(task["id"]) + + def tearDown(self): + self.clear_queue() + + def test_create_task(self): + task_id = self.queue.create_task(task_payload={"hello": "world"}) + self.assertEqual(len(self.queue), 1) + self.queue.delete_task(task_id) + self.assertEqual(len(self.queue), 0) + + def test_list_tasks(self): + num_tasks = 50 + + task_ids = set() + for i in range(num_tasks): + task_ids.add(self.queue.create_task(task_payload={"task": i})) + self.assertEqual(len(self.queue), num_tasks) + + list_task_ids = set() + list_task_payloads = set() + tasks = self.queue.list_tasks() + for task in tasks: + list_task_ids.add(task["id"]) + list_task_payloads.add(tuple(task["payload"].items())) + + expected_payloads = {tuple({"task": i}.items()) for i in range(num_tasks)} + self.assertSetEqual(list_task_ids, task_ids) + self.assertSetEqual(list_task_payloads, expected_payloads) + + for task_id in task_ids: + self.queue.delete_task(task_id) + self.assertEqual(len(self.queue), 0) + + def test_lease_tasks(self): + num_tasks = 10 + lease_length = 3600 + + task_ids = [] + for i in range(num_tasks): + task_ids.append(self.queue.create_task(task_payload={"task": i})) + self.assertEqual(len(self.queue), num_tasks) + + # Lease 1 task + expected_leaseTimestamp = (time.time() + lease_length) * 10 ** 6 + tasks = self.queue.lease_tasks(tasks_to_fetch=1, lease_secs=lease_length) + self.assertEqual(len(tasks), 1) + self.assertDictEqual(tasks[0]["payload"], {"task": 0}) + self.assertAlmostEqual(tasks[0]["leaseTimestamp"], expected_leaseTimestamp, -7) # within 10 seconds difference + + # Lease the rest of the tasks + expected_leaseTimestamp = (time.time() + lease_length) * 10 ** 6 + tasks = self.queue.lease_tasks(tasks_to_fetch=100, lease_secs=lease_length) + self.assertEqual(len(tasks), num_tasks - 1) + + payloads = set() + expected_payloads = {tuple({"task": i}.items()) for i in range(1, num_tasks)} + for task in tasks: + payloads.add(tuple(task["payload"].items())) + self.assertAlmostEqual(task["leaseTimestamp"], expected_leaseTimestamp, -7) + self.assertSetEqual(payloads, expected_payloads) + + for task_id in task_ids: + self.queue.delete_task(task_id) + self.assertEqual(len(self.queue), 0) + + def test_lease_tagged_tasks(self): + num_generic_tasks = 100 + num_tagged_tasks = 10 + lease_length = 3600 + tag = "Special" + + generic_task_ids = [] + for i in range(num_generic_tasks): + generic_task_ids.append(self.queue.create_task(task_payload={"task": i})) + self.assertEqual(len(self.queue), num_generic_tasks) + + tagged_task_ids = [] + for i in range(num_tagged_tasks): + tagged_task_ids.append(self.queue.create_task(task_payload={"tagged_task": i}, tag=tag)) + self.assertEqual(len(self.queue), num_generic_tasks + num_tagged_tasks) + + expected_leaseTimestamp = (time.time() + lease_length) * 10 ** 6 + tasks = self.queue.lease_tasks(tasks_to_fetch=1000, lease_secs=lease_length, tag=tag) + self.assertEqual(len(tasks), num_tagged_tasks) + + payloads = set() + expected_payloads = {tuple({"tagged_task": i}.items()) for i in range(num_tagged_tasks)} + for task in tasks: + payloads.add(tuple(task["payload"].items())) + self.assertAlmostEqual(task["leaseTimestamp"], expected_leaseTimestamp, -7) + self.assertSetEqual(payloads, expected_payloads) + + for task_id in generic_task_ids + tagged_task_ids: + self.queue.delete_task(task_id) + self.assertEqual(len(self.queue), 0) + + def test_update_lease(self): + payload = {"task": "update"} + task_id = self.queue.create_task(task_payload=payload) + self.assertEqual(len(self.queue), 1) + + lease_length = 3600 + expected_leaseTimestamp = (time.time() + lease_length) * 10 ** 6 + tasks = self.queue.lease_tasks(tasks_to_fetch=1000, lease_secs=lease_length) + self.assertEqual(len(tasks), 1) + self.assertDictEqual(tasks[0]["payload"], payload) + self.assertAlmostEqual(tasks[0]["leaseTimestamp"], expected_leaseTimestamp, -7) + + task_id = tasks[0]["id"] + update_lease_time = 30 + expected_leaseTimestamp = (time.time() + update_lease_time) * 10 ** 6 + task = self.queue.update_task(task_id=task_id, new_lease_secs=update_lease_time) + self.assertAlmostEqual(task["leaseTimestamp"], expected_leaseTimestamp, -7) + + self.queue.delete_task(task_id) + self.assertEqual(len(self.queue), 0) diff --git a/renderservice/render/tests/daemon/test_resource_generator.py b/renderservice/render/tests/daemon/test_resource_generator.py new file mode 100644 index 0000000..0806470 --- /dev/null +++ b/renderservice/render/tests/daemon/test_resource_generator.py @@ -0,0 +1,179 @@ +"""Test the resource generator for expected failures.""" +from render.tests.resources.BaseResourceTest import BaseResourceTest +from render.daemon.ResourceGenerator import ResourceGenerator, TaskError + + +class ResourceGeneratorTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.generator = ResourceGenerator() + self.BASE_URL = "resources/binary-cards.html" + self.TASK_TEMPLATE = { + "resource_slug": "binary-cards", + "resource_name": "Binary Cards", + "resource_view": "binary_cards", + "url": None + } + + def test_task_missing_resource_slug(self): + combination = { + "number_bits": "4", + "dot_counts": "yes", + "black_back": "yes", + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + task.pop("resource_slug") + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_task_missing_resource_name(self): + combination = { + "number_bits": "4", + "dot_counts": "yes", + "black_back": "yes", + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + task.pop("resource_name") + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_task_missing_resource_view(self): + combination = { + "number_bits": "4", + "dot_counts": "yes", + "black_back": "yes", + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + task.pop("resource_view") + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_task_missing_paper_size(self): + combination = { + "number_bits": "4", + "dot_counts": "yes", + "black_back": "yes", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_task_missing_copies(self): + combination = { + "number_bits": "4", + "dot_counts": "yes", + "black_back": "yes", + "paper_size": "a4", + "header_text": "" + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_task_missing_url(self): + combination = { + "number_bits": "4", + "dot_counts": "yes", + "black_back": "yes", + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + task = self.TASK_TEMPLATE.copy() + task.update(combination) + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_invalid_resource_view(self): + combination = { + "number_bits": "4", + "dot_counts": "yes", + "black_back": "yes", + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + task["resource_view"] = "invalid_resource.py" + with self.assertRaises(ImportError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_invalid_paper_size(self): + combination = { + "number_bits": "4", + "dot_counts": "yes", + "black_back": "yes", + "paper_size": "a0", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_invalid_copies(self): + combination = { + "number_bits": "4", + "dot_counts": "yes", + "black_back": "yes", + "paper_size": "a4", + "header_text": "", + "copies": -1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) diff --git a/renderservice/render/tests/resources/BaseResourceTest.py b/renderservice/render/tests/resources/BaseResourceTest.py new file mode 100644 index 0000000..5418cb4 --- /dev/null +++ b/renderservice/render/tests/resources/BaseResourceTest.py @@ -0,0 +1,57 @@ +"""The BaseTest case for tests to inheirit from for render tests.""" +import importlib +from render.tests.BaseTest import BaseTest +from render.daemon.ResourceGenerator import ResourceGenerator + + +class BaseResourceTest(BaseTest): + """A base test class for individual test classes.""" + + def __init__(self, *args, **kwargs): + """Create a Base Test. + + Create class inheiriting from TestCase, while also storing + the path to test files and the maxiumum difference to display on + test failures. + """ + super(BaseResourceTest, self).__init__(*args, **kwargs) + self.module = None + + @classmethod + def setUpClass(cls): + """Set up before class initialization.""" + cls.generator = ResourceGenerator() + + def load_module(self): + """Load resource module. + + Returns: + Module object to make calls from. + """ + module_path = "render.resources.{}".format(self.module) + return importlib.import_module(module_path) + + def query_string(self, values): + """Create a GET query to append to a URL from the given values. + + Args: + values: A dictionary of keys/values of GET parameters. + + Returns: + String of GET query. + """ + url_combination = {} + for parameter, value in values.items(): + if value is True: + url_combination[parameter] = "yes" + elif value is False: + url_combination[parameter] = "no" + else: + url_combination[parameter] = value + + string = "?" + for index, (key, value) in enumerate(url_combination.items()): + string += "{key}={value}".format(key=key, value=value) + if index < len(values): + string += "&" + return string diff --git a/renderservice/render/tests/resources/test_arrows.py b/renderservice/render/tests/resources/test_arrows.py new file mode 100644 index 0000000..10e19d7 --- /dev/null +++ b/renderservice/render/tests/resources/test_arrows.py @@ -0,0 +1,71 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class ArrowsResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "arrows" + self.BASE_URL = "resources/arrows.html" + self.TASK_TEMPLATE = { + "resource_slug": "arrows", + "resource_name": "Arrows", + "resource_view": "arrows", + "url": None + } + + def test_arrows_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Arrows:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_arrows_resource_generation_missing_paper_size_parameter(self): + combination = { + "header_text": "Example header text", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_arrows_resource_generation_missing_header_text_parameter(self): + expected_filename = "Arrows (a4).pdf" + combination = { + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_barcode_checksum_poster.py b/renderservice/render/tests/resources/test_barcode_checksum_poster.py new file mode 100644 index 0000000..3e827d5 --- /dev/null +++ b/renderservice/render/tests/resources/test_barcode_checksum_poster.py @@ -0,0 +1,88 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class BarcodeChecksumPosterResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "barcode_checksum_poster" + self.BASE_URL = "resources/barcode-checksum-poster.html" + self.TASK_TEMPLATE = { + "resource_slug": "barcode-checksum-poster", + "resource_name": "Barcode Checksum Poster", + "resource_view": "barcode_checksum_poster", + "url": None + } + + def test_barcode_checksum_poster_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Barcode Checksum Poster:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_barcode_checksum_poster_resource_generation_missing_barcode_length_parameter(self): + combination = { + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_barcode_checksum_poster_resource_generation_missing_paper_size_parameter(self): + combination = { + "barcode_length": "12", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_barcode_checksum_poster_resource_generation_missing_header_text_parameter(self): + expected_filename = "Barcode Checksum Poster (12 digits - a4).pdf" + combination = { + "barcode_length": "12", + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_binary_cards.py b/renderservice/render/tests/resources/test_binary_cards.py new file mode 100644 index 0000000..0d7fbfc --- /dev/null +++ b/renderservice/render/tests/resources/test_binary_cards.py @@ -0,0 +1,107 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class BinaryCardsResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "binary_cards" + self.BASE_URL = "resources/binary-cards.html" + self.TASK_TEMPLATE = { + "resource_slug": "binary-cards", + "resource_name": "Binary Cards", + "resource_view": "binary_cards", + "url": None + } + + def test_binary_cards_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Binary Cards:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_binary_cards_resource_generation_missing_numbers_parameter(self): + combination = { + "black_back": False, + "paper_size": "a4", + "header_text": "Example header text", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_binary_cards_resource_generation_missing_back_parameter(self): + combination = { + "display_numbers": True, + "paper_size": "a4", + "header_text": "Example header text", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_binary_cards_resource_generation_missing_paper_size_parameter(self): + combination = { + "display_numbers": True, + "black_back": False, + "header_text": "Example header text", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_binary_cards_resource_generation_missing_header_text_parameter(self): + expected_filename = "Binary Cards (with numbers - without black back - a4).pdf" + combination = { + "display_numbers": True, + "black_back": False, + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_binary_cards_small.py b/renderservice/render/tests/resources/test_binary_cards_small.py new file mode 100644 index 0000000..da40267 --- /dev/null +++ b/renderservice/render/tests/resources/test_binary_cards_small.py @@ -0,0 +1,145 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class BinaryCardsSmallResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "binary_cards_small" + self.BASE_URL = "resources/binary-cards-small.html" + self.TASK_TEMPLATE = { + "resource_slug": "binary-cards", + "resource_name": "Binary Cards (small)", + "resource_view": "binary_cards_small", + "url": None + } + + def test_binary_cards_small_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Binary Cards (small):") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_binary_cards_small_resource_generation_missing_dot_count_parameter(self): + combination = { + "number_bits": "4", + "black_back": True, + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_binary_cards_small_resource_generation_missing_black_back_parameter(self): + combination = { + "number_bits": "4", + "dot_counts": True, + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_binary_cards_small_resource_generation_missing_number_bits_parameter(self): + combination = { + "dot_counts": True, + "black_back": True, + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_binary_cards_small_resource_generation_missing_paper_size_parameter(self): + combination = { + "number_bits": "4", + "dot_counts": True, + "black_back": True, + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_binary_cards_small_resource_generation_missing_header_text_parameter(self): + expected_filename = "Binary Cards (small) (4 bits - with dot counts - with black back - a4).pdf" + combination = { + "dot_counts": True, + "number_bits": "4", + "black_back": True, + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) + + def test_binary_cards_small_resource_generation_invalid_black_back_parameter(self): + combination = { + "number_bits": "4", + "dot_counts": True, + "black_back": "maybe", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) diff --git a/renderservice/render/tests/resources/test_binary_to_alphabet.py b/renderservice/render/tests/resources/test_binary_to_alphabet.py new file mode 100644 index 0000000..631bbe4 --- /dev/null +++ b/renderservice/render/tests/resources/test_binary_to_alphabet.py @@ -0,0 +1,88 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class BinaryToAlphabetResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "binary_to_alphabet" + self.BASE_URL = "resources/binary-to-alphabet.html" + self.TASK_TEMPLATE = { + "resource_slug": "binary-to-alphabet", + "resource_name": "Binary To Alphabet", + "resource_view": "binary_to_alphabet", + "url": None + } + + def test_binary_to_alphabet_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Binary To Alphabet:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_binary_to_alphabet_resource_generation_missing_worksheet_version_parameter(self): + combination = { + "paper_size": "letter", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_binary_to_alphabet_resource_generation_missing_paper_size_parameter(self): + combination = { + "worksheet_version": "student", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_binary_to_alphabet_resource_generation_missing_header_text_parameter(self): + expected_filename = "Binary To Alphabet (teacher - letter).pdf" + combination = { + "worksheet_version": "teacher", + "paper_size": "letter", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_binary_windows.py b/renderservice/render/tests/resources/test_binary_windows.py new file mode 100644 index 0000000..0ab0d58 --- /dev/null +++ b/renderservice/render/tests/resources/test_binary_windows.py @@ -0,0 +1,128 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class BinaryWindowsResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "binary_windows" + self.BASE_URL = "resources/binary-windows.html" + self.TASK_TEMPLATE = { + "resource_slug": "binary-windows", + "resource_name": "Binary Windows", + "resource_view": "binary_windows", + "url": None + } + + def test_binary_windows_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Binary Windows:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_binary_windows_resource_generation_missing_dot_count_parameter(self): + combination = { + "number_bits": "8", + "value_type": "binary", + "paper_size": "a4", + "header_text": "Example header text", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_binary_windows_resource_generation_missing_number_bits_parameter(self): + combination = { + "dot_counts": True, + "value_type": "binary", + "paper_size": "a4", + "header_text": "Example header text", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_binary_windows_resource_generation_missing_value_type_parameter(self): + combination = { + "dot_counts": True, + "number_bits": "8", + "paper_size": "a4", + "header_text": "Example header text", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_binary_windows_resource_generation_missing_paper_size_parameter(self): + combination = { + "dot_counts": True, + "number_bits": "8", + "value_type": "binary", + "header_text": "Example header text", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_binary_windows_resource_generation_missing_header_text_parameter(self): + expected_filename = "Binary Windows (8 bits - lightbulb - with dot counts).pdf" + combination = { + "dot_counts": True, + "number_bits": "8", + "value_type": "lightbulb", + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_grid.py b/renderservice/render/tests/resources/test_grid.py new file mode 100644 index 0000000..ecba421 --- /dev/null +++ b/renderservice/render/tests/resources/test_grid.py @@ -0,0 +1,71 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class GridResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "grid" + self.BASE_URL = "resources/grid.html" + self.TASK_TEMPLATE = { + "resource_slug": "grid", + "resource_name": "Grid", + "resource_view": "grid", + "url": None + } + + def test_grid_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Grid:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_grid_resource_generation_missing_paper_size_parameter(self): + combination = { + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_grid_resource_generation_missing_header_text_parameter(self): + expected_filename = "Grid (a4).pdf" + combination = { + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_job_badges.py b/renderservice/render/tests/resources/test_job_badges.py new file mode 100644 index 0000000..3282c71 --- /dev/null +++ b/renderservice/render/tests/resources/test_job_badges.py @@ -0,0 +1,71 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class JobBadgesResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "job_badges" + self.BASE_URL = "resources/job-badges.html" + self.TASK_TEMPLATE = { + "resource_slug": "job-badges", + "resource_name": "Job Badges", + "resource_view": "job_badges", + "url": None + } + + def test_job_badges_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Job Badges:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_job_badges_resource_generation_missing_paper_size_parameter(self): + combination = { + "header_text": "Example header text", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_job_badges_resource_generation_missing_header_text_parameter(self): + expected_filename = "Job Badges (a4).pdf" + combination = { + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_left_right_cards.py b/renderservice/render/tests/resources/test_left_right_cards.py new file mode 100644 index 0000000..e931363 --- /dev/null +++ b/renderservice/render/tests/resources/test_left_right_cards.py @@ -0,0 +1,71 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class LeftRightCardsResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "left_right_cards" + self.BASE_URL = "resources/left-right-cards.html" + self.TASK_TEMPLATE = { + "resource_slug": "left-right-cards", + "resource_name": "Left and Right Cards", + "resource_view": "left_right_cards", + "url": None + } + + def test_left_right_cards_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Left and Right Cards:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_left_right_cards_resource_generation_missing_paper_size_parameter(self): + combination = { + "header_text": "Example header text", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_left_right_cards_resource_generation_missing_header_text_parameter(self): + expected_filename = "Left and Right Cards (a4).pdf" + combination = { + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_modulo_clock.py b/renderservice/render/tests/resources/test_modulo_clock.py new file mode 100644 index 0000000..c1e67d9 --- /dev/null +++ b/renderservice/render/tests/resources/test_modulo_clock.py @@ -0,0 +1,88 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class ModuloClockResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "modulo_clock" + self.BASE_URL = "resources/modulo-clock.html" + self.TASK_TEMPLATE = { + "resource_slug": "modulo-clock", + "resource_name": "Modulo Clock", + "resource_view": "modulo_clock", + "url": None + } + + def test_modulo_clock_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Modulo Clock:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_modulo_clock_resource_generation_missing_modulo_number_parameter(self): + combination = { + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_modulo_clock_resource_generation_missing_paper_size_parameter(self): + combination = { + "modulo_number": "2", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_modulo_clock_resource_generation_missing_header_text_parameter(self): + expected_filename = "Modulo Clock (2 - a4).pdf" + combination = { + "modulo_number": "2", + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_parity_cards.py b/renderservice/render/tests/resources/test_parity_cards.py new file mode 100644 index 0000000..a76b47e --- /dev/null +++ b/renderservice/render/tests/resources/test_parity_cards.py @@ -0,0 +1,87 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class ParityCardsResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "parity_cards" + self.BASE_URL = "resources/parity-cards.html" + self.TASK_TEMPLATE = { + "resource_slug": "parity-cards", + "resource_name": "Parity Cards", + "resource_view": "parity_cards", + "url": None + } + + def test_parity_cards_small_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Parity Cards:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_parity_cards_resource_generation_missing_back_colour_parameter(self): + combination = { + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_parity_cards_resource_generation_missing_paper_size_parameter(self): + combination = { + "back_colour": "black", + "header_text": "", + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_parity_cards_resource_generation_missing_header_text_parameter(self): + expected_filename = "Parity Cards (black back - a4).pdf" + combination = { + "back_colour": "black", + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_piano_keys.py b/renderservice/render/tests/resources/test_piano_keys.py new file mode 100644 index 0000000..371d961 --- /dev/null +++ b/renderservice/render/tests/resources/test_piano_keys.py @@ -0,0 +1,88 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class PianoKeysResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "piano_keys" + self.BASE_URL = "resources/piano-keys.html" + self.TASK_TEMPLATE = { + "resource_slug": "piano-keys", + "resource_name": "Piano Keys", + "resource_view": "piano_keys", + "url": None + } + + def test_piano_keys_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Piano Keys:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_piano_keys_resource_generation_missing_highlight_parameter(self): + combination = { + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_piano_keys_resource_generation_missing_paper_size_parameter(self): + combination = { + "highlight": False, + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_piano_keys_resource_generation_missing_header_text_parameter(self): + expected_filename = "Piano Keys (no highlight - a4).pdf" + combination = { + "highlight": False, + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_searching_cards.py b/renderservice/render/tests/resources/test_searching_cards.py new file mode 100644 index 0000000..d0758ee --- /dev/null +++ b/renderservice/render/tests/resources/test_searching_cards.py @@ -0,0 +1,128 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class SearchingCardsResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "searching_cards" + self.BASE_URL = "resources/searching-cards.html" + self.TASK_TEMPLATE = { + "resource_slug": "searching-cards", + "resource_name": "Searching Cards", + "resource_view": "searching_cards", + "url": None + } + + def test_searching_cards_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Searching Cards:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_searching_cards_resource_generation_missing_number_cards_parameter(self): + combination = { + "max_number": "99", + "help_sheet": True, + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_searching_cards_resource_generation_missing_max_number_parameter(self): + combination = { + "number_cards": "15", + "help_sheet": True, + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_searching_cards_resource_generation_missing_help_sheet_parameter(self): + combination = { + "number_cards": "15", + "max_number": "99", + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_searching_cards_resource_generation_missing_paper_size_parameter(self): + combination = { + "number_cards": "15", + "max_number": "99", + "help_sheet": True, + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_searching_cards_resource_generation_missing_header_text_parameter(self): + expected_filename = "Searching Cards (15 cards - 0 to 99 - with helper sheet - a4).pdf" + combination = { + "number_cards": "15", + "max_number": "99", + "help_sheet": True, + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_sorting_network.py b/renderservice/render/tests/resources/test_sorting_network.py new file mode 100644 index 0000000..84c6dee --- /dev/null +++ b/renderservice/render/tests/resources/test_sorting_network.py @@ -0,0 +1,88 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class SortingNetworkResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "sorting_network" + self.BASE_URL = "resources/modulo-clock.html" + self.TASK_TEMPLATE = { + "resource_slug": "sorting-network", + "resource_name": "Sorting Network", + "resource_view": "sorting_network", + "url": None + } + + def test_sorting_network_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Sorting Network:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_sorting_network_resource_generation_missing_prefilled_values_parameter(self): + combination = { + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_sorting_network_resource_generation_missing_paper_size_parameter(self): + combination = { + "prefilled_values": "blank", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_sorting_network_generation_missing_header_text_parameter(self): + expected_filename = "Sorting Network (blank - a4).pdf" + combination = { + "prefilled_values": "blank", + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_sorting_network_cards.py b/renderservice/render/tests/resources/test_sorting_network_cards.py new file mode 100644 index 0000000..7887371 --- /dev/null +++ b/renderservice/render/tests/resources/test_sorting_network_cards.py @@ -0,0 +1,88 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class SortingNetworkCardsResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "sorting_network_cards" + self.BASE_URL = "resources/sorting-network-cards.html" + self.TASK_TEMPLATE = { + "resource_slug": "sorting-network-cards", + "resource_name": "Sorting Network Cards", + "resource_view": "sorting_network_cards", + "url": None + } + + def test_sorting_network_cards_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Sorting Network Cards:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_sorting_network_cards_resource_generation_missing_type_parameter(self): + combination = { + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_sorting_network_cards_resource_generation_missing_paper_size_parameter(self): + combination = { + "type": "small_numbers", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_sorting_network_cards_resource_generation_missing_header_text_parameter(self): + expected_filename = "Sorting Network Cards (small numbers - a4).pdf" + combination = { + "type": "small_numbers", + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_train_stations.py b/renderservice/render/tests/resources/test_train_stations.py new file mode 100644 index 0000000..56a832b --- /dev/null +++ b/renderservice/render/tests/resources/test_train_stations.py @@ -0,0 +1,88 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class TrainStationsResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "train_stations" + self.BASE_URL = "resources/train-stations.html" + self.TASK_TEMPLATE = { + "resource_slug": "train-stations", + "resource_name": "Train Stations", + "resource_view": "train_stations", + "url": None + } + + def test_train_stations_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Train Stations:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_train_stations_resource_generation_missing_tracks_parameter(self): + combination = { + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_train_stations_resource_generation_missing_paper_size_parameter(self): + combination = { + "tracks": "circular", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_train_stations_resource_generation_missing_header_text_parameter(self): + expected_filename = "Train Stations (circular tracks - a4).pdf" + combination = { + "tracks": "circular", + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/resources/test_treasure_hunt.py b/renderservice/render/tests/resources/test_treasure_hunt.py new file mode 100644 index 0000000..56f540e --- /dev/null +++ b/renderservice/render/tests/resources/test_treasure_hunt.py @@ -0,0 +1,153 @@ +import itertools +from render.daemon.ResourceGenerator import TaskError +from render.tests.resources.BaseResourceTest import BaseResourceTest + + +class TreasureHuntResourceTest(BaseResourceTest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.module = "treasure_hunt" + self.BASE_URL = "resources/treasure-hunt.html" + self.TASK_TEMPLATE = { + "resource_slug": "treasure-hunt", + "resource_name": "Treasure Hunt", + "resource_view": "treasure_hunt", + "url": None + } + + def test_treasure_hunt_resource_generation_valid_configurations(self): + resource_module = self.load_module() + valid_options = resource_module.valid_options() + valid_options["header_text"] = ["", "Example header"] + valid_options["copies"] = [1] + valid_option_keys = sorted(valid_options) + + combinations = [ + dict(zip(valid_option_keys, product)) + for product in itertools.product( + *(valid_options[valid_option_key] for valid_option_key in valid_option_keys)) + ] + + print() + print("Testing Treasure Hunt:") + for combination in combinations: + print(" - Testing combination: {} ... ".format(combination), end="") + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + print("ok") + + def test_treasure_hunt_resource_generation_missing_prefilled_values_parameter(self): + combination = { + "number_order": "sorted", + "instructions": True, + "art": "colour", + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_treasure_hunt_resource_generation_missing_number_order_parameter(self): + combination = { + "prefilled_values": "blank", + "instructions": True, + "art": "colour", + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_treasure_hunt_resource_generation_missing_instructions_parameter(self): + combination = { + "number_order": "sorted", + "prefilled_values": "blank", + "art": "colour", + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_treasure_hunt_resource_generation_missing_art_parameter(self): + combination = { + "number_order": "sorted", + "prefilled_values": "blank", + "instructions": True, + "paper_size": "a4", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_treasure_hunt_resource_generation_missing_paper_size_parameter(self): + combination = { + "number_order": "sorted", + "instructions": True, + "art": "colour", + "prefilled_values": "blank", + "number_order": "sorted", + "header_text": "", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + with self.assertRaises(TaskError): + filename, pdf = self.generator.generate_resource_pdf(task) + + def test_treasure_hunt_generation_missing_header_text_parameter(self): + expected_filename = "Treasure Hunt (blank - full colour - with instructions - a4).pdf" + combination = { + "number_order": "sorted", + "instructions": True, + "art": "colour", + "prefilled_values": "blank", + "number_order": "sorted", + "paper_size": "a4", + "copies": 1 + } + + url = self.BASE_URL + self.query_string(combination) + task = self.TASK_TEMPLATE.copy() + task.update(combination) + task["url"] = url + + filename, pdf = self.generator.generate_resource_pdf(task) + self.assertEqual(filename, expected_filename) diff --git a/renderservice/render/tests/start_tests.py b/renderservice/render/tests/start_tests.py new file mode 100644 index 0000000..84eca9d --- /dev/null +++ b/renderservice/render/tests/start_tests.py @@ -0,0 +1,42 @@ +"""Starts tests for the Render Service, daemon and webservice.""" +import unittest + +# +# Resource Generation Tests +# + +from render.tests.resources.test_arrows import ArrowsResourceTest # noqa: F401 +from render.tests.resources.test_barcode_checksum_poster import BarcodeChecksumPosterResourceTest # noqa: F401 +from render.tests.resources.test_binary_cards_small import BinaryCardsSmallResourceTest # noqa: F401 +from render.tests.resources.test_binary_cards import BinaryCardsResourceTest # noqa: F401 +from render.tests.resources.test_binary_to_alphabet import BinaryToAlphabetResourceTest # noqa: F401 +from render.tests.resources.test_binary_windows import BinaryWindowsResourceTest # noqa: F401 +from render.tests.resources.test_grid import GridResourceTest # noqa: F401 +from render.tests.resources.test_job_badges import JobBadgesResourceTest # noqa: F401 +from render.tests.resources.test_left_right_cards import LeftRightCardsResourceTest # noqa: F401 +from render.tests.resources.test_modulo_clock import ModuloClockResourceTest # noqa: F401 +from render.tests.resources.test_parity_cards import ParityCardsResourceTest # noqa: F401 +from render.tests.resources.test_piano_keys import PianoKeysResourceTest # noqa: F401 +from render.tests.resources.test_searching_cards import SearchingCardsResourceTest # noqa: F401 +from render.tests.resources.test_sorting_network import SortingNetworkResourceTest # noqa: F401 +from render.tests.resources.test_sorting_network_cards import SortingNetworkCardsResourceTest # noqa: F401 +from render.tests.resources.test_train_stations import TrainStationsResourceTest # noqa: F401 +from render.tests.resources.test_treasure_hunt import TreasureHuntResourceTest # noqa: F401 + +# +# General Tests +# + +from render.tests.daemon.test_daemon_utils import DaemonUtilsTest # noqa: F401 +from render.tests.daemon.test_file_manager import FileManagerTest # noqa: F401 +from render.tests.daemon.test_resource_generator import ResourceGeneratorTest # noqa: F401 +from render.tests.daemon.test_queue_handler import QueueHandlerTest # noqa: F401 + +from render.tests.webserver.test_webserver_app import WebserverAppTest # noqa: F401 + +# +# Webservice Tests +# + +if __name__ == "__main__": + unittest.main() diff --git a/renderservice/render/tests/webserver/test_webserver_app.py b/renderservice/render/tests/webserver/test_webserver_app.py new file mode 100644 index 0000000..4181760 --- /dev/null +++ b/renderservice/render/tests/webserver/test_webserver_app.py @@ -0,0 +1,62 @@ +import unittest +import subprocess +from render.webserver.app import application +from render.daemon.utils import get_active_daemon_details, check_pid + + +class WebserverAppTest(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_index_page(self): + with application.test_client() as c: + response = c.get("/") + self.assertEqual(response.get_data().decode(), "CS-Unplugged - Render Engine") + + def test_health_check(self): + print() # print to separate logging messages + with application.test_client() as c: + response = c.get("/_ah/health") + self.assertEqual(response.status_code, 200) + + def test_health_check_recovery(self): + """This test depends on daemon utils for success.""" + print() # print to separate logging messages + # kill daemons + details = get_active_daemon_details("render") + number_daemons = len(details) + for daemon in details: + args = ["/docker_venv/bin/python", "-m", "render.daemon", + "--daemon", str(daemon.number), + "stop"] + p = subprocess.Popen(args, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + returncode = None + for _ in range(10): + try: + returncode = p.wait(1) + except subprocess.TimeoutExpired: + pass + + if returncode is None: + p.terminate() + p.wait(1) + + self.assertEqual(returncode, 0) + + # recover daemons with health check + with application.test_client() as c: + response = c.get("/_ah/health") + self.assertEqual(response.status_code, 200) + + details = get_active_daemon_details("render") + self.assertEqual(len(details), number_daemons) + + for daemon in details: + self.assertTrue(check_pid(daemon.pid)) diff --git a/renderservice/render/webserver/__init__.py b/renderservice/render/webserver/__init__.py new file mode 100644 index 0000000..0fb4fe3 --- /dev/null +++ b/renderservice/render/webserver/__init__.py @@ -0,0 +1 @@ +"""Module containing webserver logic for managing health checks and daemons.""" diff --git a/renderservice/render/webserver/app.py b/renderservice/render/webserver/app.py new file mode 100644 index 0000000..b0b9f8f --- /dev/null +++ b/renderservice/render/webserver/app.py @@ -0,0 +1,111 @@ +"""Webservice application specification.""" +import os +import logging +import subprocess +from flask import Flask +from render import __version__ +from render.daemon.utils import ( + check_pid, + get_active_daemon_details, + get_recommended_number_of_daemons + ) + +DEBUG = not int(os.getenv("FLASK_PRODUCTION", 1)) +PORT = int(os.getenv("PORT", 8080)) + +log_formatter = logging.Formatter("%(asctime)s [%(process)s] [%(levelname)s] %(message)s", "[%Y-%m-%d %H:%M:%S %z]") + +application = Flask(__name__) +logger = logging.getLogger(__name__) +log_handler = logging.StreamHandler() +log_handler.setFormatter(log_formatter) +logger.addHandler(log_handler) +logger.setLevel(logging.INFO) + + +@application.route("/") +def index(): + """Get an index page describing the service.""" + return "CS-Unplugged - Render Engine" + + +@application.route("/version") +def version(): + """Get the version of the service.""" + return __version__, 200 + + +@application.errorhandler(500) +def server_error(e): + """Log and reports back information about internal errors.""" + logger.exception("An error occurred during a request.") + return """ + An internal error occurred: 
{}
+ See logs for full stacktrace. + """.format(e), 500 + + +@application.route("/_ah/health") +def health_check(): + """Perform a health check. + + This method is to ensure the api and associated processes are + working correctly. + """ + errored = False + max_waits = 500 + + num_daemons = get_recommended_number_of_daemons() + inactive_daemons = {i for i in range(1, num_daemons + 1)} + daemons = get_active_daemon_details("render") + + # Checks which daemons are running + for info in daemons: + if check_pid(info.pid): + logger.info("Render daemon {} is running.".format(info.number)) + inactive_daemons.discard(info.number) + else: + logger.info("Render daemon {} is not running.".format(info.number)) + + if len(inactive_daemons) > 0: + logger.info("Attempprocess_taskting to restart render daemons {}.".format(inactive_daemons)) + + # Attempt to restart missing daemons + processes = [] + for daemon_number in inactive_daemons: + args = ["/docker_venv/bin/python", "-m", "render.daemon", + "--daemon", str(daemon_number), + "start"] + p = subprocess.Popen(args, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + processes.append((daemon_number, p, 0)) + + # Clean up processes created in creating daemons + while len(processes) > 0: + daemon_number, p, retries = processes.pop(0) + returncode = None + + try: + returncode = p.wait(0.02) + except subprocess.TimeoutExpired as e: + pass + + if returncode is None: + if retries < max_waits: + processes.append((daemon_number, p, retries + 1)) + else: + p.terminate() + p.wait(1) + logger.error("Failed to restart render daemon {} too many polls.".format(daemon_number, returncode)) + errored = True + continue + + if returncode != 0: + logger.error("Failed to restart render daemon {} with errorcode {}.".format(daemon_number, returncode)) + errored = True + elif returncode == 0: + logger.info("Restarted render daemon {}.".format(daemon_number)) + + return "", 200 if not errored else 500 diff --git a/renderservice/render/webserver/gunicorn.conf.py b/renderservice/render/webserver/gunicorn.conf.py new file mode 100644 index 0000000..d1c1504 --- /dev/null +++ b/renderservice/render/webserver/gunicorn.conf.py @@ -0,0 +1,13 @@ +"""Configuration file for gunicorn.""" +import multiprocessing + +# Details from https://cloud.google.com/appengine/docs/flexible/python/runtime +timeout = 10 +graceful_timeout = 30 +worker_class = 'gevent' +workers = multiprocessing.cpu_count() * 2 + 1 +forwarded_allow_ips = '*' +secure_scheme_headers = {'X-APPENGINE-HTTPS': 'on'} + +# Logging +accesslog = 'access.log' diff --git a/renderservice/render/webserver/wsgi.py b/renderservice/render/webserver/wsgi.py new file mode 100644 index 0000000..87d4b16 --- /dev/null +++ b/renderservice/render/webserver/wsgi.py @@ -0,0 +1,5 @@ +"""WSGI config for render service.""" +from render.webserver.app import application + +if __name__ == "__main__": + application.run() diff --git a/renderservice/requirements.txt b/renderservice/requirements.txt new file mode 100644 index 0000000..7761f6a --- /dev/null +++ b/renderservice/requirements.txt @@ -0,0 +1,16 @@ +flask==0.12.1 +gevent==1.2.1 +gunicorn==19.7.1 +google-cloud==0.24.0 +google-api-python-client==1.6.2 +daemons==1.3.0 +jinja2==2.9.6 +Pillow==4.1.1 +urllib3[secure]==1.21.1 +httplib2shim==0.0.2 +yattag==1.9.0 + +# Testing +coverage==4.4.1 +flake8==3.3.0 +pydocstyle==2.0.0 diff --git a/renderservice/scripts/deploy-docker-entrypoint.sh b/renderservice/scripts/deploy-docker-entrypoint.sh new file mode 100644 index 0000000..d7b30ef --- /dev/null +++ b/renderservice/scripts/deploy-docker-entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash +source /renderservice/scripts/mount-bucket.sh ${CLOUD_STORAGE_BUCKET_NAME} ${STATIC_DIRECTORY} +source /renderservice/scripts/docker-entrypoint.sh diff --git a/renderservice/scripts/docker-entrypoint.sh b/renderservice/scripts/docker-entrypoint.sh new file mode 100644 index 0000000..db2d909 --- /dev/null +++ b/renderservice/scripts/docker-entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/bash +function cpu_count(){ +/docker_venv/bin/python << END +import sys +from render.daemon.utils import get_recommended_number_of_daemons +sys.exit(get_recommended_number_of_daemons()) +END +render_daemons=$? +} + +cpu_count +for i in `seq 1 ${render_daemons}`; +do + /docker_venv/bin/python -m render.daemon --daemon ${i} start & +done + +/docker_venv/bin/gunicorn -c render/webserver/gunicorn.conf.py -b :${PORT} render.webserver.wsgi diff --git a/renderservice/scripts/mount-bucket.sh b/renderservice/scripts/mount-bucket.sh new file mode 100644 index 0000000..9937c8f --- /dev/null +++ b/renderservice/scripts/mount-bucket.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BUCKET_NAME=$1 +DIRECTORY=$2 + +if [ ! -d "${DIRECTORY}" ]; then + mkdir ${DIRECTORY} +fi + +gcsfuse --implicit-dirs --key-file ${GOOGLE_CLOUD_BUCKET_KEY} ${BUCKET_NAME} ${DIRECTORY} diff --git a/renderservice/scripts/shutdown-script.sh b/renderservice/scripts/shutdown-script.sh new file mode 100644 index 0000000..859bed4 --- /dev/null +++ b/renderservice/scripts/shutdown-script.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +MY_PROGRAM="[PROGRAM_NAME]" # For example, "apache2" or "nginx" +MY_USER="[LOCAL_USERNAME]" +CHECKPOINT="/home/$MY_USER/checkpoint.out" +GSUTIL_OPTS="-m -o GSUtil:parallel_composite_upload_threshold=32M" +BUCKET_NAME="[BUCKET_NAME]" # For example, "my-checkpoint-files" (without gs://) + +echo "Shutting down! Seeing if ${MY_PROGRAM} is running." + +# Find the newest copy of $MY_PROGRAM +PID="$(pgrep -n "$MY_PROGRAM")" + +if [[ "$?" -ne 0 ]]; then + echo "${MY_PROGRAM} not running, shutting down immediately." + exit 0 +fi + +echo "Sending SIGINT to $PID" +kill -2 "$PID" + +# Portable waitpid equivalent +while kill -0 "$PID"; do + sleep 1 +done + +echo "$PID is done, copying ${CHECKPOINT} to gs://${BUCKET_NAME} as ${MY_USER}" + +su "${MY_USER}" -c "gsutil $GSUTIL_OPTS cp $CHECKPOINT gs://${BUCKET_NAME}/" + +echo "Done uploading, shutting down." + +# TODO: +# - Signal to clear lease on tasks +# - Unmount storage + +# To Deploy +# gcloud compute instances create example-instance \ +# --metadata-from-file shutdown-script=examples/scripts/install.sh diff --git a/renderservice/static/fonts/NotoSans-Regular-LICENSE.txt b/renderservice/static/fonts/NotoSans-Regular-LICENSE.txt new file mode 100644 index 0000000..d952d62 --- /dev/null +++ b/renderservice/static/fonts/NotoSans-Regular-LICENSE.txt @@ -0,0 +1,92 @@ +This Font Software is licensed under the SIL Open Font License, +Version 1.1. + +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font +creation efforts of academic and linguistic communities, and to +provide a free and open framework in which fonts may be shared and +improved in partnership with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply to +any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software +components as distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, +deleting, or substituting -- in part or in whole -- any of the +components of the Original Version, by changing formats or by porting +the Font Software to a new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, +modify, redistribute, and sell modified and unmodified copies of the +Font Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, in +Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the +corresponding Copyright Holder. This restriction only applies to the +primary font name as presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created using +the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/renderservice/static/fonts/NotoSans-Regular.ttf b/renderservice/static/fonts/NotoSans-Regular.ttf new file mode 100644 index 0000000..04be6f5 Binary files /dev/null and b/renderservice/static/fonts/NotoSans-Regular.ttf differ diff --git a/renderservice/static/fonts/PatrickHand-LICENSE.md b/renderservice/static/fonts/PatrickHand-LICENSE.md new file mode 100644 index 0000000..f452c8f --- /dev/null +++ b/renderservice/static/fonts/PatrickHand-LICENSE.md @@ -0,0 +1,11 @@ +# Copyright + +Copyright (c) 2012 Patrick Wagesreiter ([mail@patrickwagesreiter.at](mailto:mail@patrickwagesreiter.at)) + +# License + +This Font Software is licensed under the SIL Open Font License, Version 1.1. This license is available with a FAQ at: http://scripts.sil.org/OFL + +# More info + +**Designer:** Patrick Wagesreiter http://www.patrickwagesreiter.at diff --git a/renderservice/static/fonts/PatrickHand-Regular.ttf b/renderservice/static/fonts/PatrickHand-Regular.ttf new file mode 100644 index 0000000..fb45ccd Binary files /dev/null and b/renderservice/static/fonts/PatrickHand-Regular.ttf differ diff --git a/renderservice/templates/base-resource-pdf.html b/renderservice/templates/base-resource-pdf.html new file mode 100644 index 0000000..7ac309d --- /dev/null +++ b/renderservice/templates/base-resource-pdf.html @@ -0,0 +1,43 @@ + + + {{ filename }} + + + +
+ {{ header_text }} +
+ {% for copy_data in all_data %} + {% for page in copy_data %} +
+ {% if page.type == "image" %} + + {% elif page.type == "html" %} + {% autoescape off %} + {{ page.data }} + {% endautoescape %} + {% endif %} +
+ {% endfor %} + {% endfor %} +
+ - {{ resource_name }} - {{ url }} +
+ + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..50a7dab --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +# Check Python style +flake8==3.3.0 +pydocstyle==2.0.0 + +#Coverage Tools +coverage==4.4.1 + +# Documentation +sphinx==1.6.2 +sphinx-rtd-theme==0.2.4 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..c9c2087 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,22 @@ +[flake8] +exclude = + # No need to traverse our git or venv directories + .git, + venv, + docker_venv, + + # Don't travere documentation + docs, + + # There's no value in checking cache directories + __pycache__, + +show-source = True +statistics = True +count = True +max-line-length=119 + +[pydocstyle] +# Ignore following rules to allow Google Python Style docstrings +add_ignore = D407,D413 +match_dir = (?!venv)(?!docker_venv)(?!docs)[^\.].* diff --git a/static/css/print-resource-pdf.css b/static/css/print-resource-pdf.css new file mode 100644 index 0000000..2ca89cb --- /dev/null +++ b/static/css/print-resource-pdf.css @@ -0,0 +1,446 @@ +html { + box-sizing: border-box; } + +*, +*::before, +*::after { + box-sizing: inherit; } + +body { + font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; + font-size: 1rem; + font-weight: normal; + line-height: 1.5; + color: #292b2c; + background-color: #fff; } + +[tabindex="-1"]:focus { + outline: none !important; } + +h1, h2, h3, h4, h5, h6 { + margin-top: 0; + margin-bottom: .5rem; } + +p { + margin-top: 0; + margin-bottom: 1rem; } + +address { + margin-bottom: 1rem; + font-style: normal; + line-height: inherit; } + +ol, +ul, +dl { + margin-top: 0; + margin-bottom: 1rem; } + +ol ol, +ul ul, +ol ul, +ul ol { + margin-bottom: 0; } + +dt { + font-weight: bold; } + +dd { + margin-bottom: .5rem; + margin-left: 0; } + +blockquote { + margin: 0 0 1rem; } + +a { + color: #3366ff; + text-decoration: none; } + a:focus, a:hover { + color: #0039e6; + text-decoration: underline; } + +a:not([href]):not([tabindex]) { + color: inherit; + text-decoration: none; } + a:not([href]):not([tabindex]):focus, a:not([href]):not([tabindex]):hover { + color: inherit; + text-decoration: none; } + a:not([href]):not([tabindex]):focus { + outline: 0; } + +pre { + margin-top: 0; + margin-bottom: 1rem; + overflow: auto; } + +figure { + margin: 0 0 1rem; } + +img { + vertical-align: middle; } + +table { + border-collapse: collapse; + background-color: transparent; } + +caption { + padding-top: 0.75rem; + padding-bottom: 0.75rem; + color: #636c72; + text-align: left; + caption-side: bottom; } + +th { + text-align: left; } + +label { + display: inline-block; + margin-bottom: .5rem; } + +input, +button, +select, +textarea { + line-height: inherit; } + +fieldset { + min-width: 0; + padding: 0; + margin: 0; + border: 0; } + +legend { + display: block; + width: 100%; + padding: 0; + margin-bottom: .5rem; + font-size: 1.5rem; + line-height: inherit; } + +output { + display: inline-block; } + +[hidden] { + display: none !important; } + +h1, h2, h3, h4, h5, h6, +.h1, .h2, .h3, .h4, .h5, .h6 { + margin-bottom: 0.5rem; + font-family: "Patrick Hand", cursive; + font-weight: 500; + line-height: 1.1; + color: inherit; } + +h1, .h1 { + font-size: 2.5rem; } + +h2, .h2 { + font-size: 2rem; } + +h3, .h3 { + font-size: 1.75rem; } + +h4, .h4 { + font-size: 1.5rem; } + +h5, .h5 { + font-size: 1.25rem; } + +h6, .h6 { + font-size: 1rem; } + +.lead { + font-size: 1.25rem; + font-weight: 300; } + +.display-1 { + font-size: 6rem; + font-weight: 300; + line-height: 1.1; } + +.display-2 { + font-size: 5.5rem; + font-weight: 300; + line-height: 1.1; } + +.display-3 { + font-size: 4.5rem; + font-weight: 300; + line-height: 1.1; } + +.display-4 { + font-size: 3.5rem; + font-weight: 300; + line-height: 1.1; } + +hr { + margin-top: 1rem; + margin-bottom: 1rem; + border: 0; + border-top: 1px solid rgba(0, 0, 0, 0.1); } + +small, +.small { + font-size: 80%; + font-weight: normal; } + +mark, +.mark { + padding: 0.2em; + background-color: #fcf8e3; } + +.list-unstyled { + padding-left: 0; + list-style: none; } + +.list-inline { + padding-left: 0; + list-style: none; } + +.list-inline-item { + display: inline-block; } + .list-inline-item:not(:last-child) { + margin-right: 5px; } + +.initialism { + font-size: 90%; + text-transform: uppercase; } + +.blockquote { + padding: 0.5rem 1rem; + margin-bottom: 1rem; + font-size: 1.25rem; + border-left: 0.25rem solid #eceeef; } + +.blockquote-footer { + display: block; + font-size: 80%; + color: #636c72; } + .blockquote-footer::before { + content: "\2014 \00A0"; } + +.blockquote-reverse { + padding-right: 1rem; + padding-left: 0; + text-align: right; + border-right: 0.25rem solid #eceeef; + border-left: 0; } + +.blockquote-reverse .blockquote-footer::before { + content: ""; } +.blockquote-reverse .blockquote-footer::after { + content: "\00A0 \2014"; } + +.text-lowercase { + text-transform: lowercase !important; } + +.text-uppercase { + text-transform: uppercase !important; } + +.text-capitalize { + text-transform: capitalize !important; } + +.font-weight-normal { + font-weight: normal; } + +.font-weight-bold { + font-weight: bold; } + +.font-italic { + font-style: italic; } + +.text-white { + color: #fff !important; } + +.text-muted { + color: #636c72 !important; } + +a.text-muted:focus, a.text-muted:hover { + color: #4b5257 !important; } + +.text-primary { + color: #3366ff !important; } + +a.text-primary:focus, a.text-primary:hover { + color: #0040ff !important; } + +.text-success { + color: #279f2d !important; } + +a.text-success:focus, a.text-success:hover { + color: #1d7621 !important; } + +.text-info { + color: #0d84c7 !important; } + +a.text-info:focus, a.text-info:hover { + color: #0a6497 !important; } + +.text-warning { + color: #fba428 !important; } + +a.text-warning:focus, a.text-warning:hover { + color: #ec8c04 !important; } + +.text-danger { + color: #cc0423 !important; } + +a.text-danger:focus, a.text-danger:hover { + color: #9a031a !important; } + +.text-gray-dark { + color: #292b2c !important; } + +a.text-gray-dark:focus, a.text-gray-dark:hover { + color: #101112 !important; } + +.table { + width: 100%; + max-width: 100%; + margin-bottom: 1rem; } + .table th, + .table td { + padding: 0.75rem; + vertical-align: top; + border-top: 1px solid #eceeef; } + .table thead th { + vertical-align: bottom; + border-bottom: 2px solid #eceeef; } + .table tbody + tbody { + border-top: 2px solid #eceeef; } + .table .table { + background-color: #fff; } + +.table-sm th, +.table-sm td { + padding: 0.3rem; } + +.table-bordered { + border: 1px solid #eceeef; } + .table-bordered th, + .table-bordered td { + border: 1px solid #eceeef; } + .table-bordered thead th, + .table-bordered thead td { + border-bottom-width: 2px; } + +.table-striped tbody tr:nth-of-type(odd) { + background-color: rgba(0, 0, 0, 0.05); } + +.table-hover tbody tr:hover { + background-color: rgba(0, 0, 0, 0.075); } + +.table-active, +.table-active > th, +.table-active > td { + background-color: rgba(0, 0, 0, 0.075); } + +.table-hover .table-active:hover { + background-color: rgba(0, 0, 0, 0.075); } + .table-hover .table-active:hover > td, + .table-hover .table-active:hover > th { + background-color: rgba(0, 0, 0, 0.075); } + +.table-success, +.table-success > th, +.table-success > td { + background-color: #dff0d8; } + +.table-hover .table-success:hover { + background-color: #d0e9c6; } + .table-hover .table-success:hover > td, + .table-hover .table-success:hover > th { + background-color: #d0e9c6; } + +.table-info, +.table-info > th, +.table-info > td { + background-color: #d9edf7; } + +.table-hover .table-info:hover { + background-color: #c4e3f3; } + .table-hover .table-info:hover > td, + .table-hover .table-info:hover > th { + background-color: #c4e3f3; } + +.table-warning, +.table-warning > th, +.table-warning > td { + background-color: #fcf8e3; } + +.table-hover .table-warning:hover { + background-color: #faf2cc; } + .table-hover .table-warning:hover > td, + .table-hover .table-warning:hover > th { + background-color: #faf2cc; } + +.table-danger, +.table-danger > th, +.table-danger > td { + background-color: #f2dede; } + +.table-hover .table-danger:hover { + background-color: #ebcccc; } + .table-hover .table-danger:hover > td, + .table-hover .table-danger:hover > th { + background-color: #ebcccc; } + +.thead-inverse th { + color: #fff; + background-color: #292b2c; } + +.thead-default th { + color: #464a4c; + background-color: #eceeef; } + +.table-inverse { + color: #fff; + background-color: #292b2c; } + .table-inverse th, + .table-inverse td, + .table-inverse thead th { + border-color: #fff; } + .table-inverse.table-bordered { + border: 0; } + +.table-responsive { + display: block; + width: 100%; } + .table-responsive.table-bordered { + border: 0; } + +footer { + bottom: 0; } + +header { + text-align: center; + width: 100%; + position: relative; + height: 8mm; + margin-bottom: -8mm; + font-family: 'Open Sans', sans-serif; + font-size: 0.8em; + color: #888; } + +.fixed-running-element { + text-align: center; + width: 100%; + position: fixed; + margin: 0; + font-family: 'Open Sans', sans-serif; + font-size: 0.8em; + color: #888; } + .fixed-running-element img { + max-height: 1.5em; + vertical-align: text-bottom; } + +div.page-break { + page-break-after: always; } + +div.resource-image-container { + text-align: center; } + +img.resource-image { + max-width: 100%; } + +/*# sourceMappingURL=print-resource-pdf.css.map */ diff --git a/static/img/logo-small.png b/static/img/logo-small.png new file mode 100644 index 0000000..7bf993a Binary files /dev/null and b/static/img/logo-small.png differ diff --git a/static/img/resources/arrows/arrows.png b/static/img/resources/arrows/arrows.png new file mode 100644 index 0000000..d75ac80 Binary files /dev/null and b/static/img/resources/arrows/arrows.png differ diff --git a/static/img/resources/arrows/thumbnail.png b/static/img/resources/arrows/thumbnail.png new file mode 100644 index 0000000..8f99c12 Binary files /dev/null and b/static/img/resources/arrows/thumbnail.png differ diff --git a/static/img/resources/barcode-checksum-poster/12-digits.png b/static/img/resources/barcode-checksum-poster/12-digits.png new file mode 100644 index 0000000..e8cb7e9 Binary files /dev/null and b/static/img/resources/barcode-checksum-poster/12-digits.png differ diff --git a/static/img/resources/barcode-checksum-poster/13-digits.png b/static/img/resources/barcode-checksum-poster/13-digits.png new file mode 100644 index 0000000..c6161ea Binary files /dev/null and b/static/img/resources/barcode-checksum-poster/13-digits.png differ diff --git a/static/img/resources/barcode-checksum-poster/thumbnail.gif b/static/img/resources/barcode-checksum-poster/thumbnail.gif new file mode 100644 index 0000000..de7c7f9 Binary files /dev/null and b/static/img/resources/barcode-checksum-poster/thumbnail.gif differ diff --git a/static/img/resources/binary-cards-small/binary-cards-small-1.png b/static/img/resources/binary-cards-small/binary-cards-small-1.png new file mode 100644 index 0000000..979b131 Binary files /dev/null and b/static/img/resources/binary-cards-small/binary-cards-small-1.png differ diff --git a/static/img/resources/binary-cards-small/binary-cards-small-2.png b/static/img/resources/binary-cards-small/binary-cards-small-2.png new file mode 100644 index 0000000..024f7c8 Binary files /dev/null and b/static/img/resources/binary-cards-small/binary-cards-small-2.png differ diff --git a/static/img/resources/binary-cards-small/binary-cards-small-3.png b/static/img/resources/binary-cards-small/binary-cards-small-3.png new file mode 100644 index 0000000..082d9d1 Binary files /dev/null and b/static/img/resources/binary-cards-small/binary-cards-small-3.png differ diff --git a/static/img/resources/binary-cards-small/thumbnail.png b/static/img/resources/binary-cards-small/thumbnail.png new file mode 100644 index 0000000..58e7f6c Binary files /dev/null and b/static/img/resources/binary-cards-small/thumbnail.png differ diff --git a/static/img/resources/binary-cards/binary-cards-1-dot.png b/static/img/resources/binary-cards/binary-cards-1-dot.png new file mode 100644 index 0000000..65b4e74 Binary files /dev/null and b/static/img/resources/binary-cards/binary-cards-1-dot.png differ diff --git a/static/img/resources/binary-cards/binary-cards-128-dots.png b/static/img/resources/binary-cards/binary-cards-128-dots.png new file mode 100644 index 0000000..2a8b3d8 Binary files /dev/null and b/static/img/resources/binary-cards/binary-cards-128-dots.png differ diff --git a/static/img/resources/binary-cards/binary-cards-16-dots.png b/static/img/resources/binary-cards/binary-cards-16-dots.png new file mode 100644 index 0000000..c261309 Binary files /dev/null and b/static/img/resources/binary-cards/binary-cards-16-dots.png differ diff --git a/static/img/resources/binary-cards/binary-cards-2-dots.png b/static/img/resources/binary-cards/binary-cards-2-dots.png new file mode 100644 index 0000000..ad15d7a Binary files /dev/null and b/static/img/resources/binary-cards/binary-cards-2-dots.png differ diff --git a/static/img/resources/binary-cards/binary-cards-32-dots.png b/static/img/resources/binary-cards/binary-cards-32-dots.png new file mode 100644 index 0000000..5a740ba Binary files /dev/null and b/static/img/resources/binary-cards/binary-cards-32-dots.png differ diff --git a/static/img/resources/binary-cards/binary-cards-4-dots.png b/static/img/resources/binary-cards/binary-cards-4-dots.png new file mode 100644 index 0000000..f95c273 Binary files /dev/null and b/static/img/resources/binary-cards/binary-cards-4-dots.png differ diff --git a/static/img/resources/binary-cards/binary-cards-64-dots.png b/static/img/resources/binary-cards/binary-cards-64-dots.png new file mode 100644 index 0000000..c1244ec Binary files /dev/null and b/static/img/resources/binary-cards/binary-cards-64-dots.png differ diff --git a/static/img/resources/binary-cards/binary-cards-8-dots.png b/static/img/resources/binary-cards/binary-cards-8-dots.png new file mode 100644 index 0000000..6f22e26 Binary files /dev/null and b/static/img/resources/binary-cards/binary-cards-8-dots.png differ diff --git a/static/img/resources/binary-cards/thumbnail.png b/static/img/resources/binary-cards/thumbnail.png new file mode 100644 index 0000000..f260e48 Binary files /dev/null and b/static/img/resources/binary-cards/thumbnail.png differ diff --git a/static/img/resources/binary-to-alphabet/table-teacher.png b/static/img/resources/binary-to-alphabet/table-teacher.png new file mode 100644 index 0000000..a7fec77 Binary files /dev/null and b/static/img/resources/binary-to-alphabet/table-teacher.png differ diff --git a/static/img/resources/binary-to-alphabet/table.png b/static/img/resources/binary-to-alphabet/table.png new file mode 100644 index 0000000..7d1f6c2 Binary files /dev/null and b/static/img/resources/binary-to-alphabet/table.png differ diff --git a/static/img/resources/binary-to-alphabet/thumbnail.png b/static/img/resources/binary-to-alphabet/thumbnail.png new file mode 100644 index 0000000..2b435e5 Binary files /dev/null and b/static/img/resources/binary-to-alphabet/thumbnail.png differ diff --git a/static/img/resources/binary-windows/binary-windows-1-to-8.png b/static/img/resources/binary-windows/binary-windows-1-to-8.png new file mode 100644 index 0000000..96738e6 Binary files /dev/null and b/static/img/resources/binary-windows/binary-windows-1-to-8.png differ diff --git a/static/img/resources/binary-windows/binary-windows-16-to-128.png b/static/img/resources/binary-windows/binary-windows-16-to-128.png new file mode 100644 index 0000000..935fe9b Binary files /dev/null and b/static/img/resources/binary-windows/binary-windows-16-to-128.png differ diff --git a/static/img/resources/binary-windows/binary-windows-blank.png b/static/img/resources/binary-windows/binary-windows-blank.png new file mode 100644 index 0000000..c32e75c Binary files /dev/null and b/static/img/resources/binary-windows/binary-windows-blank.png differ diff --git a/static/img/resources/binary-windows/col_binary_lightbulb.png b/static/img/resources/binary-windows/col_binary_lightbulb.png new file mode 100644 index 0000000..16e0768 Binary files /dev/null and b/static/img/resources/binary-windows/col_binary_lightbulb.png differ diff --git a/static/img/resources/binary-windows/col_binary_lightbulb_off.png b/static/img/resources/binary-windows/col_binary_lightbulb_off.png new file mode 100644 index 0000000..d8139f2 Binary files /dev/null and b/static/img/resources/binary-windows/col_binary_lightbulb_off.png differ diff --git a/static/img/resources/binary-windows/thumbnail.png b/static/img/resources/binary-windows/thumbnail.png new file mode 100644 index 0000000..a29b8ec Binary files /dev/null and b/static/img/resources/binary-windows/thumbnail.png differ diff --git a/static/img/resources/grid/thumbnail.png b/static/img/resources/grid/thumbnail.png new file mode 100644 index 0000000..0396011 Binary files /dev/null and b/static/img/resources/grid/thumbnail.png differ diff --git a/static/img/resources/job-badges/job-badges.png b/static/img/resources/job-badges/job-badges.png new file mode 100644 index 0000000..8907e21 Binary files /dev/null and b/static/img/resources/job-badges/job-badges.png differ diff --git a/static/img/resources/job-badges/thumbnail.png b/static/img/resources/job-badges/thumbnail.png new file mode 100644 index 0000000..8784a6f Binary files /dev/null and b/static/img/resources/job-badges/thumbnail.png differ diff --git a/static/img/resources/left-right-cards/left-right-cards.png b/static/img/resources/left-right-cards/left-right-cards.png new file mode 100644 index 0000000..92d2d14 Binary files /dev/null and b/static/img/resources/left-right-cards/left-right-cards.png differ diff --git a/static/img/resources/left-right-cards/thumbnail.png b/static/img/resources/left-right-cards/thumbnail.png new file mode 100644 index 0000000..0e15877 Binary files /dev/null and b/static/img/resources/left-right-cards/thumbnail.png differ diff --git a/static/img/resources/modulo-clock/modulo-clock-1.png b/static/img/resources/modulo-clock/modulo-clock-1.png new file mode 100644 index 0000000..4364c09 Binary files /dev/null and b/static/img/resources/modulo-clock/modulo-clock-1.png differ diff --git a/static/img/resources/modulo-clock/modulo-clock-10.png b/static/img/resources/modulo-clock/modulo-clock-10.png new file mode 100644 index 0000000..56f3490 Binary files /dev/null and b/static/img/resources/modulo-clock/modulo-clock-10.png differ diff --git a/static/img/resources/modulo-clock/modulo-clock-2.png b/static/img/resources/modulo-clock/modulo-clock-2.png new file mode 100644 index 0000000..ba0e2a1 Binary files /dev/null and b/static/img/resources/modulo-clock/modulo-clock-2.png differ diff --git a/static/img/resources/modulo-clock/thumbnail.png b/static/img/resources/modulo-clock/thumbnail.png new file mode 100644 index 0000000..1a5fd26 Binary files /dev/null and b/static/img/resources/modulo-clock/thumbnail.png differ diff --git a/static/img/resources/parity-cards/thumbnail.png b/static/img/resources/parity-cards/thumbnail.png new file mode 100644 index 0000000..dd5f864 Binary files /dev/null and b/static/img/resources/parity-cards/thumbnail.png differ diff --git a/static/img/resources/piano-keys/keyboard.png b/static/img/resources/piano-keys/keyboard.png new file mode 100644 index 0000000..ad3c83c Binary files /dev/null and b/static/img/resources/piano-keys/keyboard.png differ diff --git a/static/img/resources/piano-keys/thumbnail.png b/static/img/resources/piano-keys/thumbnail.png new file mode 100644 index 0000000..3b0ea2c Binary files /dev/null and b/static/img/resources/piano-keys/thumbnail.png differ diff --git a/static/img/resources/resource-sorting-network-colour.png b/static/img/resources/resource-sorting-network-colour.png new file mode 100644 index 0000000..6885816 Binary files /dev/null and b/static/img/resources/resource-sorting-network-colour.png differ diff --git a/static/img/resources/resource-sorting-network-thumbnail-example.png b/static/img/resources/resource-sorting-network-thumbnail-example.png new file mode 100644 index 0000000..4fe6b1c Binary files /dev/null and b/static/img/resources/resource-sorting-network-thumbnail-example.png differ diff --git a/static/img/resources/searching-cards/3-cards-1.png b/static/img/resources/searching-cards/3-cards-1.png new file mode 100644 index 0000000..ca12e4f Binary files /dev/null and b/static/img/resources/searching-cards/3-cards-1.png differ diff --git a/static/img/resources/searching-cards/4-cards-1.png b/static/img/resources/searching-cards/4-cards-1.png new file mode 100644 index 0000000..d74a9e3 Binary files /dev/null and b/static/img/resources/searching-cards/4-cards-1.png differ diff --git a/static/img/resources/searching-cards/4-cards-2.png b/static/img/resources/searching-cards/4-cards-2.png new file mode 100644 index 0000000..00e5d32 Binary files /dev/null and b/static/img/resources/searching-cards/4-cards-2.png differ diff --git a/static/img/resources/searching-cards/4-cards-3.png b/static/img/resources/searching-cards/4-cards-3.png new file mode 100644 index 0000000..b5c76be Binary files /dev/null and b/static/img/resources/searching-cards/4-cards-3.png differ diff --git a/static/img/resources/searching-cards/4-cards-4.png b/static/img/resources/searching-cards/4-cards-4.png new file mode 100644 index 0000000..449dbdd Binary files /dev/null and b/static/img/resources/searching-cards/4-cards-4.png differ diff --git a/static/img/resources/searching-cards/4-cards-5.png b/static/img/resources/searching-cards/4-cards-5.png new file mode 100644 index 0000000..3ca0e3a Binary files /dev/null and b/static/img/resources/searching-cards/4-cards-5.png differ diff --git a/static/img/resources/searching-cards/4-cards-6.png b/static/img/resources/searching-cards/4-cards-6.png new file mode 100644 index 0000000..416ab73 Binary files /dev/null and b/static/img/resources/searching-cards/4-cards-6.png differ diff --git a/static/img/resources/searching-cards/4-cards-7.png b/static/img/resources/searching-cards/4-cards-7.png new file mode 100644 index 0000000..ec85f3d Binary files /dev/null and b/static/img/resources/searching-cards/4-cards-7.png differ diff --git a/static/img/resources/searching-cards/thumbnail.png b/static/img/resources/searching-cards/thumbnail.png new file mode 100644 index 0000000..a74eb65 Binary files /dev/null and b/static/img/resources/searching-cards/thumbnail.png differ diff --git a/static/img/resources/sorting-network-cards/butterfly-story-baby-caterpillar.png b/static/img/resources/sorting-network-cards/butterfly-story-baby-caterpillar.png new file mode 100644 index 0000000..1e66389 Binary files /dev/null and b/static/img/resources/sorting-network-cards/butterfly-story-baby-caterpillar.png differ diff --git a/static/img/resources/sorting-network-cards/butterfly-story-butterfly.png b/static/img/resources/sorting-network-cards/butterfly-story-butterfly.png new file mode 100644 index 0000000..834ed35 Binary files /dev/null and b/static/img/resources/sorting-network-cards/butterfly-story-butterfly.png differ diff --git a/static/img/resources/sorting-network-cards/butterfly-story-caterpillar.png b/static/img/resources/sorting-network-cards/butterfly-story-caterpillar.png new file mode 100644 index 0000000..42b5a6a Binary files /dev/null and b/static/img/resources/sorting-network-cards/butterfly-story-caterpillar.png differ diff --git a/static/img/resources/sorting-network-cards/butterfly-story-leaf.png b/static/img/resources/sorting-network-cards/butterfly-story-leaf.png new file mode 100644 index 0000000..91a04ca Binary files /dev/null and b/static/img/resources/sorting-network-cards/butterfly-story-leaf.png differ diff --git a/static/img/resources/sorting-network-cards/butterfly-story-mature-pupa.png b/static/img/resources/sorting-network-cards/butterfly-story-mature-pupa.png new file mode 100644 index 0000000..bf4ff3c Binary files /dev/null and b/static/img/resources/sorting-network-cards/butterfly-story-mature-pupa.png differ diff --git a/static/img/resources/sorting-network-cards/butterfly-story-young-pupa.png b/static/img/resources/sorting-network-cards/butterfly-story-young-pupa.png new file mode 100644 index 0000000..714e8ec Binary files /dev/null and b/static/img/resources/sorting-network-cards/butterfly-story-young-pupa.png differ diff --git a/static/img/resources/sorting-network-cards/little-red-riding-hood-1.png b/static/img/resources/sorting-network-cards/little-red-riding-hood-1.png new file mode 100644 index 0000000..2a83efd Binary files /dev/null and b/static/img/resources/sorting-network-cards/little-red-riding-hood-1.png differ diff --git a/static/img/resources/sorting-network-cards/little-red-riding-hood-2.png b/static/img/resources/sorting-network-cards/little-red-riding-hood-2.png new file mode 100644 index 0000000..4b66c03 Binary files /dev/null and b/static/img/resources/sorting-network-cards/little-red-riding-hood-2.png differ diff --git a/static/img/resources/sorting-network-cards/little-red-riding-hood-3.png b/static/img/resources/sorting-network-cards/little-red-riding-hood-3.png new file mode 100644 index 0000000..fe327b5 Binary files /dev/null and b/static/img/resources/sorting-network-cards/little-red-riding-hood-3.png differ diff --git a/static/img/resources/sorting-network-cards/little-red-riding-hood-4.png b/static/img/resources/sorting-network-cards/little-red-riding-hood-4.png new file mode 100644 index 0000000..8a91a37 Binary files /dev/null and b/static/img/resources/sorting-network-cards/little-red-riding-hood-4.png differ diff --git a/static/img/resources/sorting-network-cards/little-red-riding-hood-5.png b/static/img/resources/sorting-network-cards/little-red-riding-hood-5.png new file mode 100644 index 0000000..2c5b7c1 Binary files /dev/null and b/static/img/resources/sorting-network-cards/little-red-riding-hood-5.png differ diff --git a/static/img/resources/sorting-network-cards/little-red-riding-hood-6.png b/static/img/resources/sorting-network-cards/little-red-riding-hood-6.png new file mode 100644 index 0000000..db298cf Binary files /dev/null and b/static/img/resources/sorting-network-cards/little-red-riding-hood-6.png differ diff --git a/static/img/resources/sorting-network-cards/thumbnail.png b/static/img/resources/sorting-network-cards/thumbnail.png new file mode 100644 index 0000000..61ee835 Binary files /dev/null and b/static/img/resources/sorting-network-cards/thumbnail.png differ diff --git a/static/img/resources/train-stations/thumbnail.png b/static/img/resources/train-stations/thumbnail.png new file mode 100644 index 0000000..86296f2 Binary files /dev/null and b/static/img/resources/train-stations/thumbnail.png differ diff --git a/static/img/resources/train-stations/train-stations-tracks-circular.png b/static/img/resources/train-stations/train-stations-tracks-circular.png new file mode 100644 index 0000000..9a93c82 Binary files /dev/null and b/static/img/resources/train-stations/train-stations-tracks-circular.png differ diff --git a/static/img/resources/train-stations/train-stations-tracks-twisted.png b/static/img/resources/train-stations/train-stations-tracks-twisted.png new file mode 100644 index 0000000..0e1d00e Binary files /dev/null and b/static/img/resources/train-stations/train-stations-tracks-twisted.png differ diff --git a/static/img/resources/treasure-hunt/bw.png b/static/img/resources/treasure-hunt/bw.png new file mode 100644 index 0000000..de6106d Binary files /dev/null and b/static/img/resources/treasure-hunt/bw.png differ diff --git a/static/img/resources/treasure-hunt/colour.png b/static/img/resources/treasure-hunt/colour.png new file mode 100644 index 0000000..2401ec3 Binary files /dev/null and b/static/img/resources/treasure-hunt/colour.png differ diff --git a/static/img/resources/treasure-hunt/instructions.png b/static/img/resources/treasure-hunt/instructions.png new file mode 100644 index 0000000..199e45b Binary files /dev/null and b/static/img/resources/treasure-hunt/instructions.png differ diff --git a/static/img/resources/treasure-hunt/thumbnail.png b/static/img/resources/treasure-hunt/thumbnail.png new file mode 100644 index 0000000..31c1aed Binary files /dev/null and b/static/img/resources/treasure-hunt/thumbnail.png differ diff --git a/static/scss/bootstrap/.scss-lint.yml b/static/scss/bootstrap/.scss-lint.yml new file mode 100644 index 0000000..9d6e7ec --- /dev/null +++ b/static/scss/bootstrap/.scss-lint.yml @@ -0,0 +1,548 @@ +# Default application configuration that all configurations inherit from. +scss_files: + - "**/*.scss" + - "docs/assets/scss/**/*.scss" + +exclude: + - "scss/_normalize.scss" + +plugin_directories: ['.scss-linters'] + +# List of gem names to load custom linters from (make sure they are already +# installed) +plugin_gems: [] + +# Default severity of all linters. +severity: warning + +linters: + BangFormat: + enabled: true + space_before_bang: true + space_after_bang: false + + BemDepth: + enabled: false + max_elements: 1 + + BorderZero: + enabled: true + convention: zero # or `none` + exclude: + - _normalize.scss + + ChainedClasses: + enabled: false + + ColorKeyword: + enabled: true + + ColorVariable: + enabled: false + + Comment: + enabled: true + exclude: + - _normalize.scss + - bootstrap.scss + style: silent + + DebugStatement: + enabled: true + + DeclarationOrder: + enabled: false + + DisableLinterReason: + enabled: false + + DuplicateProperty: + enabled: true + + ElsePlacement: + enabled: true + style: same_line # or 'new_line' + + EmptyLineBetweenBlocks: + enabled: false + ignore_single_line_blocks: true + + EmptyRule: + enabled: true + + ExtendDirective: + enabled: false + + FinalNewline: + enabled: true + present: true + + HexLength: + enabled: true + style: short # or 'long' + + HexNotation: + enabled: true + style: lowercase # or 'uppercase' + + HexValidation: + enabled: true + + IdSelector: + enabled: true + + ImportantRule: + enabled: false + + ImportPath: + enabled: true + leading_underscore: false + filename_extension: false + + Indentation: + enabled: true + allow_non_nested_indentation: false + character: space # or 'tab' + width: 2 + + LeadingZero: + enabled: true + style: exclude_zero # or 'include_zero' + exclude: + - _normalize.scss + + MergeableSelector: + enabled: false + force_nesting: true + + NameFormat: + enabled: true + allow_leading_underscore: true + convention: hyphenated_lowercase # or 'camel_case', or 'snake_case', or a regex pattern + + NestingDepth: + enabled: true + max_depth: 4 + ignore_parent_selectors: false + + PlaceholderInExtend: + enabled: false + + PropertyCount: + enabled: false + include_nested: false + max_properties: 10 + + PropertySortOrder: + enabled: true + ignore_unspecified: false + min_properties: 2 + separate_groups: false + exclude: + - _normalize.scss + order: + - position + - top + - right + - bottom + - left + - z-index + - -webkit-box-sizing + - -moz-box-sizing + - box-sizing + - display + - flex + - flex-align + - flex-basis + - flex-direction + - flex-wrap + - flex-flow + - flex-grow + - flex-order + - flex-pack + - align-items + - align-self + - justify-content + - float + - width + - min-width + - max-width + - height + - min-height + - max-height + - padding + - padding-top + - padding-right + - padding-bottom + - padding-left + - margin + - margin-top + - margin-right + - margin-bottom + - margin-left + - overflow + - overflow-x + - overflow-y + - -webkit-overflow-scrolling + - -ms-overflow-x + - -ms-overflow-y + - -ms-overflow-style + - clip + - clear + - font + - font-family + - font-size + - font-style + - font-weight + - font-variant + - font-size-adjust + - font-stretch + - font-effect + - font-emphasize + - font-emphasize-position + - font-emphasize-style + - font-smooth + - -webkit-hyphens + - -moz-hyphens + - hyphens + - line-height + - color + - text-align + - -webkit-text-align-last + - -moz-text-align-last + - -ms-text-align-last + - text-align-last + - text-emphasis + - text-emphasis-color + - text-emphasis-style + - text-emphasis-position + - text-decoration + - text-indent + - text-justify + - text-outline + - -ms-text-overflow + - text-overflow + - text-overflow-ellipsis + - text-overflow-mode + - text-shadow + - text-transform + - text-wrap + - -webkit-text-size-adjust + - -ms-text-size-adjust + - letter-spacing + - -ms-word-break + - word-break + - word-spacing + - -ms-word-wrap + - word-wrap + - overflow-wrap + - -moz-tab-size + - -o-tab-size + - tab-size + - white-space + - vertical-align + - list-style + - list-style-position + - list-style-type + - list-style-image + - pointer-events + - -ms-touch-action + - touch-action + - cursor + - visibility + - zoom + - table-layout + - empty-cells + - caption-side + - border-spacing + - border-collapse + - content + - quotes + - counter-reset + - counter-increment + - resize + - -webkit-user-select + - -moz-user-select + - -ms-user-select + - -o-user-select + - user-select + - nav-index + - nav-up + - nav-right + - nav-down + - nav-left + - background + - background-color + - background-image + - -ms-filter:\\'progid:DXImageTransform.Microsoft.gradient + - filter:progid:DXImageTransform.Microsoft.gradient + - filter:progid:DXImageTransform.Microsoft.AlphaImageLoader + - filter + - background-repeat + - background-attachment + - background-position + - background-position-x + - background-position-y + - -webkit-background-clip + - -moz-background-clip + - background-clip + - background-origin + - -webkit-background-size + - -moz-background-size + - -o-background-size + - background-size + - border + - border-color + - border-style + - border-width + - border-top + - border-top-color + - border-top-style + - border-top-width + - border-right + - border-right-color + - border-right-style + - border-right-width + - border-bottom + - border-bottom-color + - border-bottom-style + - border-bottom-width + - border-left + - border-left-color + - border-left-style + - border-left-width + - border-radius + - border-top-left-radius + - border-top-right-radius + - border-bottom-right-radius + - border-bottom-left-radius + - -webkit-border-image + - -moz-border-image + - -o-border-image + - border-image + - -webkit-border-image-source + - -moz-border-image-source + - -o-border-image-source + - border-image-source + - -webkit-border-image-slice + - -moz-border-image-slice + - -o-border-image-slice + - border-image-slice + - -webkit-border-image-width + - -moz-border-image-width + - -o-border-image-width + - border-image-width + - -webkit-border-image-outset + - -moz-border-image-outset + - -o-border-image-outset + - border-image-outset + - -webkit-border-image-repeat + - -moz-border-image-repeat + - -o-border-image-repeat + - border-image-repeat + - outline + - outline-width + - outline-style + - outline-color + - outline-offset + - -webkit-box-shadow + - -moz-box-shadow + - box-shadow + - filter:progid:DXImageTransform.Microsoft.Alpha(Opacity + - -ms-filter:\\'progid:DXImageTransform.Microsoft.Alpha + - opacity + - -ms-interpolation-mode + - -webkit-transition + - -moz-transition + - -ms-transition + - -o-transition + - transition + - -webkit-transition-delay + - -moz-transition-delay + - -ms-transition-delay + - -o-transition-delay + - transition-delay + - -webkit-transition-timing-function + - -moz-transition-timing-function + - -ms-transition-timing-function + - -o-transition-timing-function + - transition-timing-function + - -webkit-transition-duration + - -moz-transition-duration + - -ms-transition-duration + - -o-transition-duration + - transition-duration + - -webkit-transition-property + - -moz-transition-property + - -ms-transition-property + - -o-transition-property + - transition-property + - -webkit-transform + - -moz-transform + - -ms-transform + - -o-transform + - transform + - -webkit-transform-origin + - -moz-transform-origin + - -ms-transform-origin + - -o-transform-origin + - transform-origin + - -webkit-animation + - -moz-animation + - -ms-animation + - -o-animation + - animation + - -webkit-animation-name + - -moz-animation-name + - -ms-animation-name + - -o-animation-name + - animation-name + - -webkit-animation-duration + - -moz-animation-duration + - -ms-animation-duration + - -o-animation-duration + - animation-duration + - -webkit-animation-play-state + - -moz-animation-play-state + - -ms-animation-play-state + - -o-animation-play-state + - animation-play-state + - -webkit-animation-timing-function + - -moz-animation-timing-function + - -ms-animation-timing-function + - -o-animation-timing-function + - animation-timing-function + - -webkit-animation-delay + - -moz-animation-delay + - -ms-animation-delay + - -o-animation-delay + - animation-delay + - -webkit-animation-iteration-count + - -moz-animation-iteration-count + - -ms-animation-iteration-count + - -o-animation-iteration-count + - animation-iteration-count + - -webkit-animation-direction + - -moz-animation-direction + - -ms-animation-direction + - -o-animation-direction + + + PropertySpelling: + enabled: true + extra_properties: [] + disabled_properties: [] + + PropertyUnits: + enabled: true + global: [ + 'ch', 'em', 'ex', 'rem', # Font-relative lengths + 'cm', 'in', 'mm', 'pc', 'pt', 'px', 'q', # Absolute lengths + 'vh', 'vw', 'vmin', 'vmax', # Viewport-percentage lengths + 'deg', 'grad', 'rad', 'turn', # Angle + 'ms', 's', # Duration + 'Hz', 'kHz', # Frequency + 'dpi', 'dpcm', 'dppx', # Resolution + '%'] # Other + properties: {} + + PseudoElement: + enabled: true + + QualifyingElement: + enabled: true + allow_element_with_attribute: false + allow_element_with_class: false + allow_element_with_id: false + + SelectorDepth: + enabled: true + max_depth: 4 + + SelectorFormat: + enabled: false + convention: hyphenated_lowercase # or 'strict_BEM', or 'hyphenated_BEM', or 'snake_case', or 'camel_case', or a regex pattern + + Shorthand: + enabled: true + allowed_shorthands: [1, 2, 3, 4] + + SingleLinePerProperty: + enabled: false + allow_single_line_rule_sets: true + + SingleLinePerSelector: + enabled: false + + SpaceAfterComma: + enabled: false + style: one_space # or 'no_space', or 'at_least_one_space' + + SpaceAfterPropertyColon: + enabled: true + style: at_least_one_space # or 'no_space', or 'at_least_one_space', or 'aligned' + + SpaceAfterPropertyName: + enabled: true + + SpaceAfterVariableName: + enabled: true + + SpaceAroundOperator: + enabled: true + style: one_space # or 'at_least_one_space', or 'no_space' + + SpaceBeforeBrace: + enabled: true + style: space # or 'new_line' + allow_single_line_padding: true + + SpaceBetweenParens: + enabled: true + spaces: 0 + + StringQuotes: + enabled: true + style: double_quotes # or double_quotes + + TrailingSemicolon: + enabled: true + + TrailingWhitespace: + enabled: true + + TrailingZero: + enabled: false + + TransitionAll: + enabled: false + + UnnecessaryMantissa: + enabled: true + + UnnecessaryParentReference: + enabled: true + + UrlFormat: + enabled: true + + UrlQuotes: + enabled: true + + VariableForProperty: + enabled: false + properties: [] + + VendorPrefix: + enabled: true + identifier_list: base + additional_identifiers: [] + excluded_identifiers: [] + exclude: + - _normalize.scss + + ZeroUnit: + enabled: true + + Compass::*: + enabled: false diff --git a/static/scss/bootstrap/_alert.scss b/static/scss/bootstrap/_alert.scss new file mode 100644 index 0000000..d9b4e9b --- /dev/null +++ b/static/scss/bootstrap/_alert.scss @@ -0,0 +1,55 @@ +// +// Base styles +// + +.alert { + padding: $alert-padding-y $alert-padding-x; + margin-bottom: $alert-margin-bottom; + border: $alert-border-width solid transparent; + @include border-radius($alert-border-radius); +} + +// Headings for larger alerts +.alert-heading { + // Specified to prevent conflicts of changing $headings-color + color: inherit; +} + +// Provide class for links that match alerts +.alert-link { + font-weight: $alert-link-font-weight; +} + + +// Dismissible alerts +// +// Expand the right padding and account for the close button's positioning. + +.alert-dismissible { + // Adjust close link position + .close { + position: relative; + top: -$alert-padding-y; + right: -$alert-padding-x; + padding: $alert-padding-y $alert-padding-x; + color: inherit; + } +} + + +// Alternate styles +// +// Generate contextual modifier classes for colorizing the alert. + +.alert-success { + @include alert-variant($alert-success-bg, $alert-success-border, $alert-success-text); +} +.alert-info { + @include alert-variant($alert-info-bg, $alert-info-border, $alert-info-text); +} +.alert-warning { + @include alert-variant($alert-warning-bg, $alert-warning-border, $alert-warning-text); +} +.alert-danger { + @include alert-variant($alert-danger-bg, $alert-danger-border, $alert-danger-text); +} diff --git a/static/scss/bootstrap/_badge.scss b/static/scss/bootstrap/_badge.scss new file mode 100644 index 0000000..e5a3298 --- /dev/null +++ b/static/scss/bootstrap/_badge.scss @@ -0,0 +1,77 @@ +// Base class +// +// Requires one of the contextual, color modifier classes for `color` and +// `background-color`. + +.badge { + display: inline-block; + padding: $badge-padding-y $badge-padding-x; + font-size: $badge-font-size; + font-weight: $badge-font-weight; + line-height: 1; + color: $badge-color; + text-align: center; + white-space: nowrap; + vertical-align: baseline; + @include border-radius(); + + // Empty badges collapse automatically + &:empty { + display: none; + } +} + +// Quick fix for badges in buttons +.btn .badge { + position: relative; + top: -1px; +} + +// scss-lint:disable QualifyingElement +// Add hover effects, but only for links +a.badge { + @include hover-focus { + color: $badge-link-hover-color; + text-decoration: none; + cursor: pointer; + } +} +// scss-lint:enable QualifyingElement + +// Pill badges +// +// Make them extra rounded with a modifier to replace v3's badges. + +.badge-pill { + padding-right: $badge-pill-padding-x; + padding-left: $badge-pill-padding-x; + @include border-radius($badge-pill-border-radius); +} + +// Colors +// +// Contextual variations (linked badges get darker on :hover). + +.badge-default { + @include badge-variant($badge-default-bg); +} + +.badge-primary { + @include badge-variant($badge-primary-bg); +} + +.badge-success { + @include badge-variant($badge-success-bg); +} + +.badge-info { + @include badge-variant($badge-info-bg); +} + +.badge-warning { + @include badge-variant($badge-warning-bg); +} + +.badge-danger { + @include badge-variant($badge-danger-bg); +} diff --git a/static/scss/bootstrap/_breadcrumb.scss b/static/scss/bootstrap/_breadcrumb.scss new file mode 100644 index 0000000..1a09bba --- /dev/null +++ b/static/scss/bootstrap/_breadcrumb.scss @@ -0,0 +1,38 @@ +.breadcrumb { + padding: $breadcrumb-padding-y $breadcrumb-padding-x; + margin-bottom: $spacer-y; + list-style: none; + background-color: $breadcrumb-bg; + @include border-radius($border-radius); + @include clearfix; +} + +.breadcrumb-item { + float: left; + + // The separator between breadcrumbs (by default, a forward-slash: "/") + + .breadcrumb-item::before { + display: inline-block; // Suppress underlining of the separator in modern browsers + padding-right: $breadcrumb-item-padding; + padding-left: $breadcrumb-item-padding; + color: $breadcrumb-divider-color; + content: "#{$breadcrumb-divider}"; + } + + // IE9-11 hack to properly handle hyperlink underlines for breadcrumbs built + // without `