From 2bda0b6d003b0e4359dc0705ec13c2a4623542dd Mon Sep 17 00:00:00 2001 From: Tom Hamilton Stubber Date: Mon, 22 Jun 2020 14:13:02 +0100 Subject: [PATCH] Heroku ready (#276) * Get heroku ready * Add yml file * Remove unneeded files * Remove deploy files * Remove check for services being ready * Update logs * Update logging * Updating tc root * Logging for photos * Logging for setting an image * More logging * Images to aws * Remove hash test * Storing images in s3 and tests * Adding secrets * Tests * Add runtime * Fix deploy * Install boto3 * Fix images * Remove end2end --- .editorconfig | 31 ---------- .github/workflows/ci.yml | 3 + .pyup.yml | 1 - Makefile | 11 ++-- activate.dev.sh | 8 --- client.py | 51 ++++------------- deploy/compose | 15 ----- deploy/create-machine | 22 ------- deploy/deploy | 4 -- docker-compose.beta.yml | 17 ------ docker-compose.override.yml | 15 ----- docker-compose.prod.yml | 17 ------ docker-compose.yml | 76 ------------------------ heroku.yml | 4 ++ nginx/dev/Dockerfile | 2 - nginx/dev/nginx.conf | 49 ---------------- nginx/prod/Dockerfile | 5 -- nginx/prod/allowed.nginx.conf | 26 --------- nginx/prod/nginx.conf | 92 ------------------------------ nginx/test-keys/cert.pem | 16 ------ nginx/test-keys/key.pem | 16 ------ runtime.txt | 1 + start-prod.sh | 6 -- tcsocket/Dockerfile | 4 +- tcsocket/app/geo.py | 2 +- tcsocket/app/logs.py | 8 +-- tcsocket/app/management.py | 4 +- tcsocket/app/middleware.py | 2 +- tcsocket/app/processing.py | 11 ++-- tcsocket/app/settings.py | 25 ++++---- tcsocket/app/views/appointments.py | 2 +- tcsocket/app/views/company.py | 2 +- tcsocket/app/views/contractor.py | 4 +- tcsocket/app/views/enquiry.py | 2 +- tcsocket/app/worker.py | 42 ++++++++------ tcsocket/requirements.txt | 3 +- tcsocket/run.py | 50 ++++++---------- tests/conftest.py | 8 +-- tests/end2end.sh | 43 -------------- tests/requirements.txt | 1 + tests/test_public_views.py | 8 +-- tests/test_set_contractor.py | 50 ++++++++++++---- tests/test_utils.py | 2 +- 43 files changed, 154 insertions(+), 607 deletions(-) delete mode 100644 .editorconfig delete mode 100644 .pyup.yml delete mode 100644 activate.dev.sh delete mode 100755 deploy/compose delete mode 100755 deploy/create-machine delete mode 100755 deploy/deploy delete mode 100644 docker-compose.beta.yml delete mode 100644 docker-compose.override.yml delete mode 100644 docker-compose.prod.yml delete mode 100644 docker-compose.yml create mode 100644 heroku.yml delete mode 100644 nginx/dev/Dockerfile delete mode 100644 nginx/dev/nginx.conf delete mode 100644 nginx/prod/Dockerfile delete mode 100644 nginx/prod/allowed.nginx.conf delete mode 100644 nginx/prod/nginx.conf delete mode 100644 nginx/test-keys/cert.pem delete mode 100644 nginx/test-keys/key.pem create mode 100644 runtime.txt delete mode 100755 start-prod.sh delete mode 100755 tests/end2end.sh diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index b7ed6e26..00000000 --- a/.editorconfig +++ /dev/null @@ -1,31 +0,0 @@ -root = true - -[*] -end_of_line = lf -insert_final_newline = true -line_length = 120 -trim_trailing_whitespace = true - -[*.js,*.json,*.css,*.scss,*.sass] -indent_style = space -indent_size = 2 - -[*.jinja,*.jinja2,*.html,*.htm,*.vue,*.svg] -indent_style = spaceg -indent_size = 2 - -[*.txt,*.yml,*.yaml,*.conf,*.nginx] -indent_style = space -indent_size = 2 - -[*.py] -indent_style = space -indent_size = 4 - -[Makefile] -indent_style = tab - -[*.md,*.rst] -indent_style = space -indent_size = 4 - diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fe25001..cc612150 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,9 @@ jobs: - name: test run: make test + env: + AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }} + AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} - name: codecov run: bash <(curl -s https://codecov.io/bash) diff --git a/.pyup.yml b/.pyup.yml deleted file mode 100644 index 2e26c6d4..00000000 --- a/.pyup.yml +++ /dev/null @@ -1 +0,0 @@ -schedule: 'every month' diff --git a/Makefile b/Makefile index fd178668..68ba11e2 100644 --- a/Makefile +++ b/Makefile @@ -24,9 +24,10 @@ lint: test: pytest --cov=tcsocket -.PHONY: testcov -testcov: test - coverage html +.PHONY: build +build: + docker build tcsocket/ -t tcsocket -.PHONY: all -all: testcov lint +.PHONY: prod-push +prod-push: + git push heroku `git rev-parse --abbrev-ref HEAD`:master diff --git a/activate.dev.sh b/activate.dev.sh deleted file mode 100644 index a04b2f52..00000000 --- a/activate.dev.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -. env/bin/activate -export COMMIT=`git rev-parse HEAD` -export RELEASE_DATE="" -export APP_MASTER_KEY="testing" -export CLIENT_SIGNING_KEY="testing" -export COMPOSE_PROJECT_NAME='socket' -export RAVEN_DSN="-" diff --git a/client.py b/client.py index 677b7b55..fc5f1f4e 100755 --- a/client.py +++ b/client.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3.6 import asyncio import json import os @@ -109,10 +108,7 @@ async def company_options(*, public_key, **kwargs): 'last_name': 'Howell', 'town': 'Edinburgh', 'country': 'United Kingdom', - 'location': { - 'latitude': None, - 'longitude': None - }, + 'location': {'latitude': None, 'longitude': None}, 'photo': 'http://unsplash.com/photos/vltMzn0jqsA/download', 'extra_attributes': [ { @@ -121,9 +117,9 @@ async def company_options(*, public_key, **kwargs): 'type': 'text_extended', 'sort_index': 0, 'value': 'The returned group is itself an iterator that shares the underlying iterable with groupby(). ' - 'Because the source is shared, when the groupby() object is advanced, the previous group is no ' - 'longer visible. So, if that data is needed later, it should be stored as a list:', - 'id': 195 + 'Because the source is shared, when the groupby() object is advanced, the previous group is no ' + 'longer visible. So, if that data is needed later, it should be stored as a list:', + 'id': 195, }, { 'machine_name': None, @@ -131,45 +127,20 @@ async def company_options(*, public_key, **kwargs): 'type': 'text_short', 'sort_index': 0, 'value': 'Harvard', - 'id': 196 + 'id': 196, }, ], 'skills': [ - { - 'qual_level': 'A Level', - 'subject': 'Mathematics', - 'qual_level_ranking': 18.0, - 'category': 'Maths' - }, - { - 'qual_level': 'GCSE', - 'subject': 'Mathematics', - 'qual_level_ranking': 16.0, - 'category': 'Maths' - }, - { - 'qual_level': 'GCSE', - 'subject': 'Algebra', - 'qual_level_ranking': 16.0, - 'category': 'Maths' - }, - { - 'qual_level': 'KS3', - 'subject': 'Language', - 'qual_level_ranking': 13.0, - 'category': 'English' - }, - { - 'qual_level': 'Degree', - 'subject': 'Mathematics', - 'qual_level_ranking': 21.0, - 'category': 'Maths' - }, + {'qual_level': 'A Level', 'subject': 'Mathematics', 'qual_level_ranking': 18.0, 'category': 'Maths'}, + {'qual_level': 'GCSE', 'subject': 'Mathematics', 'qual_level_ranking': 16.0, 'category': 'Maths'}, + {'qual_level': 'GCSE', 'subject': 'Algebra', 'qual_level_ranking': 16.0, 'category': 'Maths'}, + {'qual_level': 'KS3', 'subject': 'Language', 'qual_level_ranking': 13.0, 'category': 'English'}, + {'qual_level': 'Degree', 'subject': 'Mathematics', 'qual_level_ranking': 21.0, 'category': 'Maths'}, ], 'labels': [], 'last_updated': '2017-01-08T12:20:46.244Z', 'created': '2015-01-19', - 'release_timestamp': '2017-01-08T12:27:07.541165Z' + 'release_timestamp': '2017-01-08T12:27:07.541165Z', } diff --git a/deploy/compose b/deploy/compose deleted file mode 100755 index 71d24bc7..00000000 --- a/deploy/compose +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -e -if [ "$MODE" != "PRODUCTION" ] && [ "$MODE" != "BETA" ] ; then - echo "MODE not set to PRODUCTION or BETA, use 'source activate.prod.sh'" - exit 2 -fi -if [ ! -v PG_AUTH_PASS ] ; then - echo '$PG_AUTH_PASS must be set before running compose in production or beta mode' - exit 1 -fi -export COMMIT="`git symbolic-ref --short HEAD`:`git rev-parse HEAD`" -export RELEASE_DATE=`date` -the_command="docker-compose -f docker-compose.yml -f ${COMPOSE_EXTRA:-docker-compose.prod.yml} $@" -echo "Running '${the_command}'..." -eval ${the_command} diff --git a/deploy/create-machine b/deploy/create-machine deleted file mode 100755 index 94c44bb7..00000000 --- a/deploy/create-machine +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${AWS_ACCESS_KEY+x} ]; then - echo 'Error: $AWS_ACCESS_KEY not set' - exit 2 -fi - -if [ -z ${AWS_SECRET_KEY+x} ]; then - echo 'Error: $AWS_SECRET_KEY not set' - exit 2 -fi - - - -docker-machine create \ - --driver amazonec2 \ - --amazonec2-access-key $AWS_ACCESS_KEY \ - --amazonec2-secret-key $AWS_SECRET_KEY \ - --amazonec2-instance-type t2.medium \ - --amazonec2-root-size 50 \ - --amazonec2-region eu-west-1 \ - socket diff --git a/deploy/deploy b/deploy/deploy deleted file mode 100755 index 51f24997..00000000 --- a/deploy/deploy +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -set -e -THIS_DIR=$(dirname "$0") -eval "${THIS_DIR}/compose up -d --build $@" diff --git a/docker-compose.beta.yml b/docker-compose.beta.yml deleted file mode 100644 index d5c9bda2..00000000 --- a/docker-compose.beta.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '2.1' - -services: - logs: - entrypoint: - - '/bin/logspout' - - '${LOGSPOUT_ENDPOINT}' - - nginx: - build: nginx/prod - volumes: - - media:/media:ro - restart: always - ports: - - 443:443 - depends_on: - - web diff --git a/docker-compose.override.yml b/docker-compose.override.yml deleted file mode 100644 index bc27260c..00000000 --- a/docker-compose.override.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: '2.1' -services: - logs: - ports: - - 5001:80 - - nginx: - build: nginx/dev - restart: always - volumes: - - media:/media:ro - ports: - - 5000:80 - depends_on: - - web diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml deleted file mode 100644 index da714273..00000000 --- a/docker-compose.prod.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '2.1' - -services: - logs: - entrypoint: - - '/bin/logspout' - - '${LOGSPOUT_ENDPOINT}' - - nginx: - build: nginx/prod - volumes: - - media:/media:ro - restart: always - ports: - - 443:443 - depends_on: - - logs diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 63aeeed0..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,76 +0,0 @@ -version: '2.1' - -volumes: - pg_data: {} - media: {} - redis_data: {} - -services: - logs: - image: gliderlabs/logspout - environment: - SYSLOG_HOSTNAME: ${SERVER_NAME:-tcsocket} - volumes: - - /var/run/docker.sock:/var/run/docker.sock - restart: always - - postgres: - image: postgres:9.6-alpine - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: docker - volumes: - - pg_data:/var/lib/postgresql/data - restart: always - depends_on: - - logs - - redis: - image: redis:4-alpine - volumes: - - redis_data:/data - restart: always - depends_on: - - logs - - web: - build: tcsocket - environment: - APP_PG_HOST: postgres - APP_PG_PASSWORD: docker - APP_REDIS_HOST: redis - APP_MASTER_KEY: ${APP_MASTER_KEY} - APP_TC_API_ROOT: ${TC_API_ROOT} - APP_GEOCODING_KEY: ${GEOCODING_KEY} - BIND: 0.0.0.0:8000 - COMMIT: ${COMMIT} - RELEASE_DATE: ${RELEASE_DATE} - RAVEN_DSN: ${RAVEN_DSN} - SERVER_NAME: ${SERVER_NAME:-tcsocket} - restart: always - depends_on: - - postgres - - redis - - worker: - build: tcsocket - volumes: - - media:/media - entrypoint: - - ./run.py - - worker - environment: - APP_PG_HOST: postgres - APP_PG_PASSWORD: docker - APP_REDIS_HOST: redis - APP_MEDIA_DIR: /media - APP_TC_API_ROOT: ${TC_API_ROOT} - APP_GRECAPTCHA_SECRET: ${GRECAPTCHA_SECRET} - APP_GEOCODING_KEY: ${GEOCODING_KEY} - COMMIT: ${COMMIT} - RAVEN_DSN: ${RAVEN_DSN} - SERVER_NAME: ${SERVER_NAME:-tcsocket} - restart: always - depends_on: - - postgres - - redis diff --git a/heroku.yml b/heroku.yml new file mode 100644 index 00000000..69f26524 --- /dev/null +++ b/heroku.yml @@ -0,0 +1,4 @@ +build: + docker: + web: tcsocket/Dockerfile + worker: tcsocket/Dockerfile diff --git a/nginx/dev/Dockerfile b/nginx/dev/Dockerfile deleted file mode 100644 index dd6d6ee9..00000000 --- a/nginx/dev/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM nginx:1.13-alpine -ADD ./nginx.conf /etc/nginx/nginx.conf diff --git a/nginx/dev/nginx.conf b/nginx/dev/nginx.conf deleted file mode 100644 index c7dca6cd..00000000 --- a/nginx/dev/nginx.conf +++ /dev/null @@ -1,49 +0,0 @@ -worker_processes 1; - -user nobody nogroup; -pid /tmp/nginx.pid; -error_log /dev/stdout crit; - -events { - worker_connections 1024; # increase if you have lots of clients - accept_mutex off; # set to 'on' if nginx worker_processes > 1 - # 'use epoll;' to enable for Linux 2.6+ - # 'use kqueue;' to enable for FreeBSD, OSX -} - -http { - include mime.types; - # fallback in case we can't determine a type - default_type application/octet-stream; - - log_format custom '$remote_addr request="$request" status=$status time=${request_time}s ' - 'request_size=$request_length response_size=$body_bytes_sent ' - 'referrer="$http_referer"'; - access_log /dev/stdout custom; - sendfile on; - - upstream app_server { - # fail_timeout=0 means we always retry an upstream even if it failed - # to return a good HTTP response - # for a TCP configuration - server web:8000 fail_timeout=0; - } - - server { - listen 80 default_server; - - client_max_body_size 4G; - - keepalive_timeout 5; - - location /media { - alias /media; - } - - location / { - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_pass http://app_server; - } - } -} diff --git a/nginx/prod/Dockerfile b/nginx/prod/Dockerfile deleted file mode 100644 index a0139655..00000000 --- a/nginx/prod/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM nginx:1.13-alpine -ADD ./nginx.conf /etc/nginx/nginx.conf -ADD ./allowed.nginx.conf /etc/nginx/allowed.nginx.conf -ADD ./keys/cert.pem /ssl/cert.pem -ADD ./keys/key.pem /ssl/key.pem diff --git a/nginx/prod/allowed.nginx.conf b/nginx/prod/allowed.nginx.conf deleted file mode 100644 index 0f654ee5..00000000 --- a/nginx/prod/allowed.nginx.conf +++ /dev/null @@ -1,26 +0,0 @@ -## address from https://www.cloudflare.com/ips/ -# set which addressse are allowed, -# unfortunately this doesn't seem to play nicely with set_real_ip_from -allow 103.21.244.0/22; -allow 103.22.200.0/22; -allow 103.31.4.0/22; -allow 104.16.0.0/12; -allow 108.162.192.0/18; -allow 131.0.72.0/22; -allow 141.101.64.0/18; -allow 162.158.0.0/15; -allow 172.64.0.0/13; -allow 173.245.48.0/20; -allow 188.114.96.0/20; -allow 190.93.240.0/20; -allow 197.234.240.0/22; -allow 198.41.128.0/17; -allow 199.27.128.0/21; -allow 2400:cb00::/32; -allow 2405:8100::/32; -allow 2405:b500::/32; -allow 2606:4700::/32; -allow 2803:f800::/32; -allow 2c0f:f248::/32; -allow 2a06:98c0::/29; -deny all; diff --git a/nginx/prod/nginx.conf b/nginx/prod/nginx.conf deleted file mode 100644 index eeea6207..00000000 --- a/nginx/prod/nginx.conf +++ /dev/null @@ -1,92 +0,0 @@ -worker_processes 1; - -user nobody nogroup; -pid /tmp/nginx.pid; -error_log /dev/stdout crit; - -events { - worker_connections 1024; # increase if you have lots of clients - accept_mutex off; # set to 'on' if nginx worker_processes > 1 - use epoll; -} - -http { - include mime.types; - # fallback in case we can't determine a type - default_type application/octet-stream; - - log_format custom '$http_x_forwarded_for request="$request" status=$status time=${request_time}s ' - 'request_size=$request_length response_size=$body_bytes_sent ' - 'referrer="$http_referer"'; - access_log /dev/stdout custom; - sendfile on; - - upstream app_server { - # fail_timeout=0 means we always retry an upstream even if it failed - # to return a good HTTP response - # for a TCP configuration - server web:8000 fail_timeout=0; - } - - server { - # if no Host match, close the connection - listen 443 ssl http2 default_server; - - ssl on; - ssl_certificate /ssl/cert.pem; - ssl_certificate_key /ssl/key.pem; - - return 444; - } - - server { - listen 443 ssl http2; - server_name *.tutorcruncher.com; - include /etc/nginx/allowed.nginx.conf; - - ssl on; - ssl_certificate /ssl/cert.pem; - ssl_certificate_key /ssl/key.pem; - - client_max_body_size 4G; - - keepalive_timeout 5; - - location /media { - alias /media; - } - - location / { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header Host $http_host; - - # we don't want nginx trying to do something clever with - # redirects, we set the Host: header above already. - proxy_redirect off; - proxy_pass http://app_server; - } - } - - server { - # used for getting nginx stats internally only - listen 80; - server_name nginx; - - keepalive_timeout 5; - access_log off; - - location /status { - stub_status on; - } - - location / { - proxy_set_header X-Request-Start $msec; - proxy_redirect off; - proxy_send_timeout 3; - proxy_read_timeout 3; - proxy_connect_timeout 3; - proxy_pass http://app_server; - } - } -} diff --git a/nginx/test-keys/cert.pem b/nginx/test-keys/cert.pem deleted file mode 100644 index 7bb52164..00000000 --- a/nginx/test-keys/cert.pem +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICcDCCAdmgAwIBAgIJAMhIKt8Q4xDRMA0GCSqGSIb3DQEBCwUAMFExCzAJBgNV -BAYTAkdCMQowCAYDVQQIDAEtMQowCAYDVQQHDAEtMQowCAYDVQQKDAEtMQowCAYD -VQQLDAEtMRIwEAYDVQQDDAlsb2NhbGhvc3QwHhcNMTcwMTIzMTgxNzQ3WhcNMTgw -MTIzMTgxNzQ3WjBRMQswCQYDVQQGEwJHQjEKMAgGA1UECAwBLTEKMAgGA1UEBwwB -LTEKMAgGA1UECgwBLTEKMAgGA1UECwwBLTESMBAGA1UEAwwJbG9jYWxob3N0MIGf -MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC1MaYkQ10QvqZSkyKAgGbP643yQflY -X3XgS8UYSpjVBC21gdYJuGpglImpOBWpIp3P7SZEmvompaaBYIM2/Pm0veafxm3x -NTXCF2xqzVl84uLch/m9UHJYJuzQbDRb3lfOPCi1sawINeSoXqIIZnL3iC3exnQs -m2yqFR7QW9iGdwIDAQABo1AwTjAdBgNVHQ4EFgQU4d4gwwpzmb331fm+A7edvDPp -FrYwHwYDVR0jBBgwFoAU4d4gwwpzmb331fm+A7edvDPpFrYwDAYDVR0TBAUwAwEB -/zANBgkqhkiG9w0BAQsFAAOBgQBgpw/3mCFeL3jNIjogzIAojkxxIPjqkWRk07w8 -atTkLW/owXRS/G1mKruoxFWfjPej7irl0si87+1J1Mb54XDcnddpkwycwsKf3hId -BJyMSlWv/Pia4+ZGuxVwVRRSJUJfEiu05RC1X6QaV4P7/2DWlu8CaAOa8bgx7BG6 -wIAatw== ------END CERTIFICATE----- diff --git a/nginx/test-keys/key.pem b/nginx/test-keys/key.pem deleted file mode 100644 index 90a91d7f..00000000 --- a/nginx/test-keys/key.pem +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBALUxpiRDXRC+plKT -IoCAZs/rjfJB+VhfdeBLxRhKmNUELbWB1gm4amCUiak4Fakinc/tJkSa+ialpoFg -gzb8+bS95p/GbfE1NcIXbGrNWXzi4tyH+b1Qclgm7NBsNFveV848KLWxrAg15Khe -oghmcveILd7GdCybbKoVHtBb2IZ3AgMBAAECgYEAga5UI5YW8JoSvfzSb7f2XY9Q -W3e+dvJzkiGIobcaJTNFZ6fSQci8Uf1dfUBHuHKvEhbAEc/9g1WBkw05PEzYhcAe -PaMU8ZtIydfP+0kXhlwqj31mTGqUR7dGUhNX7Z00HBnTUZoiWLWKQh+ee0bj5bkP -R96wHaQsZ9cqhODoa1kCQQDbs5hgmpHkx393ck4zoL7zS23uTgBz9mrzLb4z07lJ -2XaQo9hVXlAYNcQlNKW0dyDu2xKqCrRL1vnHxmtMma5tAkEA0yFaL5waO6lV0R1p -mWrDaAZQvZs+wi81pX+cZ1YzLeo4cMpUaljOEALu473JGULx9C9/EuCRtGKkNFXj -rdqp8wJALaN7LeYuFGZU1k1KbXMg941dwrk1YuF3ihiggEelH/AqrxU6JVG4Na9F -la0AFyMAFl4v3F7o4TBBJvzS4VCzaQJAGZ/IpFKb1HXe1nxtXpNYl/18OybmXQcB -yc5NGzWZDI+KvhWwXR/eEo9okvdruscnOm2xTIc459249Ckgcu9BBQJAcTlp6NRc -EzHs3lSC/T+b3s5tYVjsTmbD5bfYM+IqQk8C9H5M2ZkJd2TAYsOKV1BApl6Vm06x -iRO13s8hNOeUEA== ------END PRIVATE KEY----- diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 00000000..385705b5 --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +python-3.8.3 diff --git a/start-prod.sh b/start-prod.sh deleted file mode 100755 index 62406d78..00000000 --- a/start-prod.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -gnome-terminal \ - --geometry=180x40 \ - --tab-with-profile=prod \ - -- bash -c "cd $(dirname "$0"); exec bash --rcfile ${1:-activate.prod.sh}" diff --git a/tcsocket/Dockerfile b/tcsocket/Dockerfile index 0d8c3db2..b998edbd 100644 --- a/tcsocket/Dockerfile +++ b/tcsocket/Dockerfile @@ -11,5 +11,5 @@ ADD ./run.py /home/root/run.py ENV PYTHONUNBUFFERED 1 WORKDIR /home/root -ENTRYPOINT ["./run.py"] -CMD ["web"] +ENTRYPOINT ["python ./run.py"] +CMD ["auto"] diff --git a/tcsocket/app/geo.py b/tcsocket/app/geo.py index 90e2be1e..8557d6b0 100644 --- a/tcsocket/app/geo.py +++ b/tcsocket/app/geo.py @@ -9,7 +9,7 @@ IP_HEADER = 'X-Forwarded-For' COUNTRY_HEADER = 'CF-IPCountry' MAX_GEOCODE_PER_HOUR = 20 -logger = logging.getLogger('socket.geo') +logger = logging.getLogger('socket') def get_ip(request): diff --git a/tcsocket/app/logs.py b/tcsocket/app/logs.py index cf3ca835..b9e821a5 100644 --- a/tcsocket/app/logs.py +++ b/tcsocket/app/logs.py @@ -15,9 +15,9 @@ def setup_logging(verbose: bool = False): config = { 'version': 1, 'disable_existing_loggers': False, - 'formatters': {'socket.default': {'format': '%(levelname)s %(name)s %(message)s'}}, + 'formatters': {'socket': {'format': '%(levelname)s %(name)s %(message)s'}}, 'handlers': { - 'socket.default': {'level': log_level, 'class': 'logging.StreamHandler', 'formatter': 'socket.default'}, + 'socket': {'level': log_level, 'class': 'logging.StreamHandler', 'formatter': 'socket'}, 'sentry': { 'level': 'WARNING', 'class': 'raven.handlers.logging.SentryHandler', @@ -27,9 +27,9 @@ def setup_logging(verbose: bool = False): }, }, 'loggers': { - 'socket': {'handlers': ['socket.default', 'sentry'], 'level': log_level}, + 'socket': {'handlers': ['socket', 'sentry'], 'level': log_level}, 'gunicorn.error': {'handlers': ['sentry'], 'level': 'ERROR'}, - 'arq': {'handlers': ['socket.default', 'sentry'], 'level': log_level}, + 'arq': {'handlers': ['socket', 'sentry'], 'level': log_level}, }, } logging.config.dictConfig(config) diff --git a/tcsocket/app/management.py b/tcsocket/app/management.py index 3ffdda58..4050abfb 100644 --- a/tcsocket/app/management.py +++ b/tcsocket/app/management.py @@ -9,7 +9,7 @@ from .models import Base, sa_companies from .settings import Settings -logger = logging.getLogger('socket.management') +logger = logging.getLogger('socket') SQL_PREPARE = """ @@ -34,7 +34,7 @@ def lenient_connection(settings: Settings, retries=5): try: - return psycopg2.connect(password=settings.pg_password, dsn=settings.pg_dsn,) + return psycopg2.connect(password=settings.pg_password, dsn=settings.pg_dsn) except psycopg2.Error as e: if retries <= 0: raise diff --git a/tcsocket/app/middleware.py b/tcsocket/app/middleware.py index 491301b3..82ba199d 100644 --- a/tcsocket/app/middleware.py +++ b/tcsocket/app/middleware.py @@ -16,7 +16,7 @@ from .utils import HTTPBadRequestJson, HTTPForbiddenJson, HTTPNotFoundJson, HTTPUnauthorizedJson from .validation import VIEW_MODELS -request_logger = logging.getLogger('socket.request') +request_logger = logging.getLogger('socket') PUBLIC_VIEWS = { 'index', diff --git a/tcsocket/app/processing.py b/tcsocket/app/processing.py index 345e76bb..cfa5dc5b 100644 --- a/tcsocket/app/processing.py +++ b/tcsocket/app/processing.py @@ -10,7 +10,7 @@ from .utils import HTTPForbiddenJson, HTTPNotFoundJson from .validation import ContractorModel, ExtraAttributeModel -logger = logging.getLogger('socket.processing') +logger = logging.getLogger('socket') def _distinct(iter, key): @@ -121,7 +121,7 @@ async def contractor_set( :param skip_deleted: whether or not to skip deleted contractors (or delete them in the db.) :return: Action: created, updated or deleted """ - from .worker import get_image + from .worker import process_image if contractor.deleted: if not skip_deleted: @@ -180,11 +180,10 @@ async def contractor_set( await _set_labels(conn, company['id'], contractor.labels) if contractor.photo: # Sometimes creating the contractor is already done on a job, so don't need another one. + job_kwargs = dict(company_key=company['public_key'], contractor_id=contractor.id, url=contractor.photo) if redis: - await redis.enqueue_job( - 'get_image', company_key=company['public_key'], contractor_id=contractor.id, url=contractor.photo - ) + await redis.enqueue_job('process_image', **job_kwargs) else: - await get_image(ctx, company_key=company['public_key'], contractor_id=contractor.id, url=contractor.photo) + await process_image(ctx, **job_kwargs) logger.info('%s contractor on %s', r.action, company['public_key']) return r.action diff --git a/tcsocket/app/settings.py b/tcsocket/app/settings.py index 4c8a27e4..4cd54533 100644 --- a/tcsocket/app/settings.py +++ b/tcsocket/app/settings.py @@ -10,14 +10,15 @@ class Settings(BaseSettings): - pg_dsn: Optional[str] = 'postgres://postgres@localhost:5432/socket' + pg_dsn: Optional[str] = 'postgresql://postgres@localhost:5432/socket' redis_settings: RedisSettings = 'redis://localhost:6379' redis_database: int = 0 master_key = b'this is a secret' - media_dir = Path('./media') - media_url = '/media' + aws_access_key: Optional[str] = 'testing' + aws_secret_key: Optional[str] = 'testing' + aws_bucket_name: str = 'socket-images-beta.tutorcruncher.com' tc_api_root = 'https://secure.tutorcruncher.com/api' grecaptcha_secret = 'required secret for google recaptcha' grecaptcha_url = 'https://www.google.com/recaptcha/api/siteverify' @@ -28,14 +29,6 @@ class Settings(BaseSettings): tc_enquiry_endpoint = '/enquiry/' tc_book_apt_endpoint = '/recipient_appointments/' - @validator('media_dir') - def check_media_dir(cls, p): - path = p.resolve() - path.mkdir(parents=True, exist_ok=True) - if not path.is_dir(): - raise ValueError(f'"{path}" is not a directory') - return str(path) - @validator('redis_settings', always=True, pre=True) def parse_redis_settings(cls, v): conf = urlparse(v) @@ -43,6 +36,10 @@ def parse_redis_settings(cls, v): host=conf.hostname, port=conf.port, password=conf.password, database=int((conf.path or '0').strip('/')), ) + @property + def images_url(self): + return f'https://{self.aws_bucket_name}' + @property def _pg_dsn_parsed(self): return urlparse(self.pg_dsn) @@ -65,5 +62,11 @@ def pg_port(self): class Config: fields = { + 'port': {'env': 'PORT'}, 'pg_dsn': {'env': 'DATABASE_URL'}, + 'redis_settings': {'env': 'REDISCLOUD_URL'}, + 'tc_api_root': {'env': 'TC_API_ROOT'}, + 'aws_access_key': {'env': 'AWS_ACCESS_KEY'}, + 'aws_secret_key': {'env': 'AWS_SECRET_KEY'}, + 'aws_bucket_name': {'env': 'AWS_BUCKET_NAME'}, } diff --git a/tcsocket/app/views/appointments.py b/tcsocket/app/views/appointments.py index e9eaefff..6725ecfd 100644 --- a/tcsocket/app/views/appointments.py +++ b/tcsocket/app/views/appointments.py @@ -25,7 +25,7 @@ ) from ..validation import AppointmentModel, BookingModel -logger = logging.getLogger('socket.views') +logger = logging.getLogger('socket') apt_c = sa_appointments.c ser_c = sa_services.c diff --git a/tcsocket/app/views/company.py b/tcsocket/app/views/company.py index 8808ecd0..aac2830a 100644 --- a/tcsocket/app/views/company.py +++ b/tcsocket/app/views/company.py @@ -7,7 +7,7 @@ from ..utils import HTTPConflictJson, json_response from ..validation import CompanyCreateModal, CompanyOptionsModel, CompanyUpdateModel -logger = logging.getLogger('socket.views') +logger = logging.getLogger('socket') async def company_create(request): diff --git a/tcsocket/app/views/contractor.py b/tcsocket/app/views/contractor.py index ecab08cd..4fa6e272 100644 --- a/tcsocket/app/views/contractor.py +++ b/tcsocket/app/views/contractor.py @@ -12,7 +12,7 @@ from ..utils import HTTPNotFoundJson, get_arg, get_pagination, json_response, route_url, slugify from ..validation import ContractorModel -logger = logging.getLogger('socket.views') +logger = logging.getLogger('socket') async def contractor_set(request): @@ -54,7 +54,7 @@ def _get_name(name_display, row): def _photo_url(request, con, thumb): ext = '.thumb.jpg' if thumb else '.jpg' - return f'{request.app["settings"].media_url}/{request["company"].public_key}/{con.id}{ext}?h={con.photo_hash}' + return f'{request.app["settings"].images_url}/{request["company"].public_key}/{con.id}{ext}?h={con.photo_hash}' async def contractor_list(request): # noqa: C901 (ignore complexity) diff --git a/tcsocket/app/views/enquiry.py b/tcsocket/app/views/enquiry.py index 3fd259d7..54362905 100644 --- a/tcsocket/app/views/enquiry.py +++ b/tcsocket/app/views/enquiry.py @@ -12,7 +12,7 @@ from ..utils import HTTPBadRequestJson, json_response from ..worker import REDIS_ENQUIRY_CACHE_KEY, get_enquiry_options, store_enquiry_data -logger = logging.getLogger('socket.views') +logger = logging.getLogger('socket') VISIBLE_FIELDS = 'client_name', 'client_email', 'client_phone', 'service_recipient_name' diff --git a/tcsocket/app/worker.py b/tcsocket/app/worker.py index 1b32a66b..c43d8491 100644 --- a/tcsocket/app/worker.py +++ b/tcsocket/app/worker.py @@ -4,11 +4,12 @@ import logging import os from datetime import datetime, timedelta -from pathlib import Path +from io import BytesIO from signal import SIGTERM from tempfile import TemporaryFile from urllib.parse import urlencode +import boto3 from aiohttp import ClientSession from aiopg.sa import create_engine from arq import create_pool, cron @@ -20,6 +21,7 @@ from .middleware import domain_allowed from .models import sa_appointments, sa_contractors from .processing import contractor_set +from .settings import Settings from .validation import ContractorModel CHUNK_SIZE = int(1e4) @@ -28,7 +30,7 @@ REDIS_ENQUIRY_CACHE_KEY = b'enquiry-data-%d' CT_JSON = 'application/json' -logger = logging.getLogger('socket.worker') +logger = logging.getLogger('socket') async def store_enquiry_data(redis, company, data): @@ -36,7 +38,7 @@ async def store_enquiry_data(redis, company, data): async def startup(ctx, retries=5): - if ctx.get('session') and ctx.get('media') and ctx.get('pg_engine'): + if ctx.get('session') and ctx.get('pg_engine'): # happens if startup is called twice eg. in test setup return try: @@ -51,7 +53,6 @@ async def startup(ctx, retries=5): else: logger.info('db engine created successfully') ctx['session'] = ClientSession() - ctx['media'] = Path(ctx['settings'].media_dir) async def shutdown(ctx): @@ -64,11 +65,9 @@ async def shutdown(ctx): await session.close() -async def get_image(ctx, company_key, contractor_id, url): - save_dir = Path(ctx['settings'].media_dir) / company_key - save_dir.mkdir(exist_ok=True) - image_path_main = save_dir / f'{contractor_id}.jpg' - image_path_thumb = save_dir / f'{contractor_id}.thumb.jpg' +async def process_image(ctx, company_key, contractor_id, url): + image_path_main = f'{company_key}/{contractor_id}.jpg' + image_path_thumb = f'{company_key}/{contractor_id}.thumb.jpg' with TemporaryFile() as f: async with ctx['session'].get(url) as r: if r.status != 200: @@ -81,10 +80,8 @@ async def get_image(ctx, company_key, contractor_id, url): if not chunk: break f.write(chunk) + image_hash = save_image(ctx['settings'], f, image_path_main, image_path_thumb) - save_image(f, image_path_main, image_path_thumb) - - image_hash = hashlib.md5(image_path_thumb.read_bytes()).hexdigest() async with ctx['pg_engine'].acquire() as conn: await conn.execute( update(sa_contractors).values(photo_hash=image_hash[:6]).where(sa_contractors.c.id == contractor_id) @@ -231,7 +228,7 @@ async def kill_worker(ctx): class WorkerSettings: - functions = [get_image, submit_booking, submit_enquiry, update_contractors, update_enquiry_options] + functions = [process_image, submit_booking, submit_enquiry, update_contractors, update_enquiry_options] cron_jobs = [ cron(delete_old_appointments, hour={0, 3, 6, 9, 12, 15, 18, 21}, minute=0), cron(kill_worker, hour=3, minute=0), @@ -248,8 +245,13 @@ class WorkerSettings: } -def save_image(file, image_path_main, image_path_thumb): +def save_image(settings: Settings, file, image_path_main, image_path_thumb): file.seek(0) + if not settings.aws_access_key: + return + s3_client = boto3.client( + 's3', aws_access_key_id=settings.aws_access_key, aws_secret_access_key=settings.aws_secret_key + ) with Image.open(file) as img: # could use more of https://piexif.readthedocs.io/en/latest/sample.html#rotate-image-by-exif-orientation if hasattr(img, '_getexif'): @@ -261,7 +263,15 @@ def save_image(file, image_path_main, image_path_thumb): img = img.convert('RGB') img_large = ImageOps.fit(img, SIZE_LARGE, Image.LANCZOS) - img_large.save(image_path_main, 'JPEG') + with BytesIO() as temp_file: + img_large.save(temp_file, format='JPEG', optimize=True) + temp_file.seek(0) + s3_client.upload_fileobj(Fileobj=temp_file, Bucket=settings.aws_bucket_name, Key=image_path_main) img_thumb = ImageOps.fit(img, SIZE_SMALL, Image.LANCZOS) - img_thumb.save(image_path_thumb, 'JPEG') + with BytesIO() as temp_file: + img_thumb.save(temp_file, format='JPEG', optimize=True) + temp_file.seek(0) + s3_client.upload_fileobj(Fileobj=temp_file, Bucket=settings.aws_bucket_name, Key=image_path_thumb) + + return hashlib.md5(img_thumb.tobytes()).hexdigest() diff --git a/tcsocket/requirements.txt b/tcsocket/requirements.txt index c772767c..9dbe2ab9 100644 --- a/tcsocket/requirements.txt +++ b/tcsocket/requirements.txt @@ -2,7 +2,8 @@ SQLAlchemy==1.3.17 aiodns==2.0.0 aiohttp==3.6.2 aiopg==1.0.0 -arq==0.18.4 +arq==0.19.0 +boto3==1.14.7 cchardet==2.1.6 gunicorn==20.0.4 python-dateutil==2.8.1 diff --git a/tcsocket/run.py b/tcsocket/run.py index 805d5d69..8a0aa94d 100755 --- a/tcsocket/run.py +++ b/tcsocket/run.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3.6 import asyncio import logging import os @@ -14,7 +13,7 @@ from app.settings import Settings from app.worker import WorkerSettings -logger = logging.getLogger('socket.run') +logger = logging.getLogger('socket') @click.group() @@ -26,29 +25,6 @@ def cli(verbose): setup_logging(verbose) -async def _check_port_open(host, port, loop): - steps, delay = 100, 0.1 - for i in range(steps): - try: - await loop.create_connection(lambda: asyncio.Protocol(), host=host, port=port) - except OSError: - await asyncio.sleep(delay, loop=loop) - else: - logger.info('Connected successfully to %s:%s after %0.2fs', host, port, delay * i) - return - raise RuntimeError(f'Unable to connect to {host}:{port} after {steps * delay}s') - - -def check_services_ready(): - settings = Settings() - loop = asyncio.get_event_loop() - coros = [ - _check_port_open(settings.pg_host, settings.pg_port, loop), - _check_port_open(settings.redis_host, settings.redis_port, loop), - ] - loop.run_until_complete(asyncio.gather(*coros, loop=loop)) - - def check_app(): loop = asyncio.get_event_loop() logger.info("initialising aiohttp app to check it's working...") @@ -60,16 +36,12 @@ def check_app(): logger.info('app started and stopped successfully, apparently configured correctly') -@cli.command() def web(): """ Serve the application If the database doesn't already exist it will be created. """ - logger.info('waiting for postgres and redis to come up...') - check_services_ready() - logger.info('preparing the database...') prepare_database(False) @@ -95,17 +67,33 @@ def load(self): Application().run() -@cli.command() def worker(): """ Run the worker """ logger.info('waiting for redis to come up...') - check_services_ready() settings = Settings() run_worker(WorkerSettings, redis_settings=settings.redis_settings, ctx={'settings': settings}) +@cli.command() +def auto(): + port_env = os.getenv('PORT') + dyno_env = os.getenv('DYNO') + if dyno_env: + logger.info('using environment variable DYNO=%r to infer command', dyno_env) + if dyno_env.lower().startswith('web'): + web() + else: + worker() + elif port_env and port_env.isdigit(): + logger.info('using environment variable PORT=%s to infer command as web', port_env) + web() + else: + logger.info('no environment variable found to infer command, assuming worker') + worker() + + @cli.command() @click.option('--no-input', is_flag=True) def resetdb(no_input): diff --git a/tests/conftest.py b/tests/conftest.py index 612a0d25..586f007f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,7 +26,7 @@ from tcsocket.app.worker import WorkerSettings, startup MASTER_KEY = 'this is the master key' -DB_DSN = 'postgres://postgres@localhost:5432/socket_test' +DB_DSN = 'postgresql://postgres@localhost:5432/socket_test' async def test_image_view(request): @@ -350,7 +350,8 @@ async def _fix_worker(redis, worker_ctx): yield worker - worker.pool = None + # Sets the pool to use our settings RedisSettings instead of ArqRedis + worker._pool = None await worker.close() @@ -378,13 +379,12 @@ def image_download_url(other_server): @pytest.fixture -def settings(tmpdir, other_server): +def settings(other_server): return Settings( pg_dsn=os.getenv('DATABASE_URL', DB_DSN), redis_database=7, master_key=MASTER_KEY, grecaptcha_secret='X' * 30, - media_dir=str(tmpdir / 'media'), grecaptcha_url=f'http://localhost:{other_server.port}/grecaptcha', tc_api_root=f'http://localhost:{other_server.port}/api', geocoding_url=f'http://localhost:{other_server.port}/geocode', diff --git a/tests/end2end.sh b/tests/end2end.sh deleted file mode 100755 index c99d8e3b..00000000 --- a/tests/end2end.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# script to test end to end use of tutorcruncher-socket -# should be run from the project root directory -set -e -set -x -# use dummy ssl certs to test -# this makes sure keys doesn't already exist -key_dir="nginx/prod/keys" -if [[ ! -e ${key_dir} ]]; then - echo "key dir ${key_dir} does not exist, creating it and using dummy keys" - mkdir ${key_dir} - cp nginx/test-keys/* ${key_dir} -elif [[ ! -d ${key_dir} ]]; then - echo "key dir ${key_dir} already exists" -fi - -# prevent the cloudflare ip check returning 403 below -echo "allow all;" > nginx/prod/allowed.nginx.conf -export LOGSPOUT_ENDPOINT="syslog://example.com:8000" -export APP_MASTER_KEY="123" -export SERVER_NAME="end2endtest" -export MODE="PRODUCTION" -export RAVEN_DSN="-" -export PG_AUTH_PASS="testing" - -docker version -docker info - -deploy/deploy - -sleep 10 - -docker ps -docker-compose ps - -# the first curl prints the response -# the second curl fails if the response is not ok -curl -kv -H "Host: socket.tutorcruncher.com" https://localhost:443 -docker-compose logs -t - -curl -kfs -H "Host: socket.tutorcruncher.com" https://localhost:443 > /dev/null - -printf "\n\nend to end tests successful.\n" diff --git a/tests/requirements.txt b/tests/requirements.txt index edd1c640..31bfae0c 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -12,3 +12,4 @@ pytest-isort==1.0.0 pytest-mock==3.1.0 pytest-sugar==0.9.3 pytest-toolbox==0.4 +requests==2.24.0 diff --git a/tests/test_public_views.py b/tests/test_public_views.py index c033ba7f..c6dc66ad 100644 --- a/tests/test_public_views.py +++ b/tests/test_public_views.py @@ -35,7 +35,7 @@ async def test_favicon(cli, mocker): assert middleware.log_warning.call_count == 0 -async def test_list_contractors(cli, db_conn): +async def test_list_contractors(cli, db_conn, settings): v = await db_conn.execute( sa_companies.insert() .values(name='testing', public_key='thepublickey', private_key='theprivatekey') @@ -65,7 +65,7 @@ async def test_list_contractors(cli, db_conn): 'id': 1, 'link': '1-fred-b', 'name': 'Fred B', - 'photo': '/media/thepublickey/1.thumb.jpg?h=abc', + 'photo': f'{settings.images_url}/thepublickey/1.thumb.jpg?h=abc', 'tag_line': None, 'primary_description': None, 'town': None, @@ -126,7 +126,7 @@ async def test_json_encoding(cli, db_conn, company, headers, newline_count): assert (await r.text()).count('\n') == newline_count -async def test_get_contractor(cli, db_conn): +async def test_get_contractor(cli, db_conn, settings): v = await db_conn.execute( sa_companies.insert() .values(name='testing', public_key='thepublickey', private_key='theprivatekey') @@ -160,7 +160,7 @@ async def test_get_contractor(cli, db_conn): 'extra_attributes': [{'sort_index': 1, 'foo': 'spam'}, {'sort_index': 5, 'foo': 'bar'}, {'foo': 'apple'}], 'labels': [], 'tag_line': None, - 'photo': '/media/thepublickey/1.jpg?h=-', + 'photo': f'{settings.images_url}/thepublickey/1.jpg?h=-', 'primary_description': None, 'review_duration': None, 'review_rating': None, diff --git a/tests/test_set_contractor.py b/tests/test_set_contractor.py index c61c312e..3e8cc836 100644 --- a/tests/test_set_contractor.py +++ b/tests/test_set_contractor.py @@ -1,10 +1,13 @@ import hashlib import hmac import json +from io import BytesIO from pathlib import Path from time import time +import boto3 import pytest +import requests from PIL import Image from tcsocket.app.models import sa_con_skills, sa_contractors, sa_labels, sa_qual_levels, sa_subjects @@ -284,8 +287,28 @@ async def test_extra_attributes_null(cli, db_conn, company): assert result.primary_description is None +def fake_s3_client(tmpdir): + class FakeS3Client: + def __init__(self, *args, **kwargs): + self.tmpdir = tmpdir + + def upload_fileobj(self, Fileobj: BytesIO, Bucket: str, Key: str): + split_key = Key.split('/') + p_company, p_file = split_key[-2], split_key[-1] + path = Path(self.tmpdir / p_company) + path.mkdir(exist_ok=True) + + with open(Path(path / p_file), 'wb+') as f: + f.write(Fileobj.getbuffer()) + + return FakeS3Client + + @pytest.mark.parametrize('image_format', ['JPEG', 'RGBA', 'P']) -async def test_photo(cli, db_conn, company, image_download_url, tmpdir, other_server, image_format, worker): +async def test_photo( + monkeypatch, cli, db_conn, company, image_download_url, tmpdir, other_server, image_format, worker +): + monkeypatch.setattr(boto3, 'client', fake_s3_client(tmpdir)) r = await signed_request( cli, f'/{company.public_key}/webhook/contractor', @@ -298,19 +321,20 @@ async def test_photo(cli, db_conn, company, image_download_url, tmpdir, other_se assert other_server.app['request_log'] == [('test_image', image_format)] assert [cs.first_name async for cs in await db_conn.execute(sa_contractors.select())] == ['Fred'] - path = Path(tmpdir / 'media' / company.public_key / '123.jpg') + path = Path(tmpdir / company.public_key / '123.jpg') assert path.exists() with Image.open(str(path)) as im: assert im.size == (1000, 1000) assert im.getpixel((1, 1)) == (128, 128, 128) - path = Path(tmpdir / 'media' / company.public_key / '123.thumb.jpg') + path = Path(tmpdir / company.public_key / '123.thumb.jpg') assert path.exists() with Image.open(str(path)) as im: assert im.size == (256, 256) assert im.getpixel((1, 1)) == (128, 128, 128) -async def test_photo_rotation(cli, db_conn, company, image_download_url, tmpdir, other_server, worker): +async def test_photo_rotation(monkeypatch, cli, db_conn, company, image_download_url, tmpdir, other_server, worker): + monkeypatch.setattr(boto3, 'client', fake_s3_client(tmpdir)) r = await signed_request( cli, f'/{company.public_key}/webhook/contractor', @@ -323,12 +347,12 @@ async def test_photo_rotation(cli, db_conn, company, image_download_url, tmpdir, assert other_server.app['request_log'] == [('test_image', None)] assert [cs.first_name async for cs in await db_conn.execute(sa_contractors.select())] == ['Fred'] - path = Path(tmpdir / 'media' / company.public_key / '123.jpg') + path = Path(tmpdir / company.public_key / '123.jpg') assert path.exists() with Image.open(str(path)) as im: assert im.size == (1000, 1000) assert im.getpixel((1, 1)) == (50, 100, 149) # image has been rotated - path = Path(tmpdir / 'media' / company.public_key / '123.thumb.jpg') + path = Path(tmpdir / company.public_key / '123.thumb.jpg') assert path.exists() with Image.open(str(path)) as im: assert im.size == (256, 256) @@ -346,7 +370,7 @@ async def test_update(cli, db_conn, company): assert [cs.first_name async for cs in await db_conn.execute(sa_contractors.select())] == ['George'] -async def test_photo_hash(cli, db_conn, company, image_download_url, tmpdir, worker): +async def test_real_s3_test(cli, db_conn, company, image_download_url, tmpdir, worker, settings): r = await signed_request(cli, f'/{company.public_key}/webhook/contractor', id=123, first_name='Fred') assert r.status == 201, await r.text() await worker.run_check() @@ -364,11 +388,13 @@ async def test_photo_hash(cli, db_conn, company, image_download_url, tmpdir, wor assert r.status == 201, await r.text() await worker.run_check() - path = Path(tmpdir / 'media' / company.public_key / '124.thumb.jpg') - assert path.exists() - - cons = sorted([(cs.first_name, cs.photo_hash) async for cs in await db_conn.execute(sa_contractors.select())]) - assert cons == [('Fred', '-'), ('George', hashlib.md5(path.read_bytes()).hexdigest()[:6])] + # Checking URL is accessible + r = requests.get(f'{settings.images_url}/{company.public_key}/124.jpg') + assert r.status_code == 200 + s3_client = boto3.Session(aws_access_key_id=settings.aws_access_key, aws_secret_access_key=settings.aws_secret_key) + bucket = s3_client.resource('s3').Bucket(settings.aws_bucket_name) + r = bucket.objects.filter(Prefix=f'{company.public_key}/').delete() + assert len(r[0].get('Deleted')) == 2 async def test_delete(cli, db_conn, company): diff --git a/tests/test_utils.py b/tests/test_utils.py index 5d4dfb51..76b9aea1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -83,7 +83,7 @@ async def test_500_error(aiohttp_client, caplog): r = await client.get('/') assert r.status == 500 assert '500: Internal Server Error' == await r.text() - assert 'ERROR socket.request:middleware.py' in caplog.text + assert 'ERROR socket:middleware.py' in caplog.text async def test_401_return_error(aiohttp_client, mocker):