From b61287ad6e47be71fc9f6a6864cdb2a04de1a81d Mon Sep 17 00:00:00 2001 From: "Simson L. Garfinkel" Date: Sat, 28 Dec 2024 22:55:04 +0000 Subject: [PATCH 01/22] make installation work on aws linux --- Makefile | 23 +++++++++++++++-------- docs/configuring_aws.rst | 5 +++++ 2 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 docs/configuring_aws.rst diff --git a/Makefile b/Makefile index 3c66e87f..a1c38aba 100644 --- a/Makefile +++ b/Makefile @@ -199,8 +199,8 @@ jscoverage: ################################################################ -# Installations are used by the CI pipeline: -# Generic: +# Installations are used by the CI pipeline and by developers +# $(REQ) gets made by the virtual environment installer, but you need to have python installed first. PLANTTRACER_LOCALDB_NAME ?= actions_test create_localdb: @@ -226,16 +226,24 @@ install-chromium-browser-macos: $(REQ) # Includes ubuntu dependencies install-ubuntu: $(REQ) echo on GitHub, we use this action instead: https://github.com/marketplace/actions/setup-ffmpeg - make venv which ffmpeg || sudo apt install ffmpeg which node || sudo apt-get install nodejs which npm || sudo apt-get install npm npm ci - make venv if [ -r requirements-ubuntu.txt ]; then $(PIP_INSTALL) -r requirements-ubuntu.txt ; fi +# Install for AWS Linux for running SAM +install-aws: + echo install for AWS Linux, for making the lambda. + echo note does not install ffmpeg currently + sudo dnf install -y python3.11 + sudo dnf install -y nodejs npm + npm ci + make $(REQ) + if [ -r requirements-aws.txt ]; then $(PIP_INSTALL) -r requirements-ubuntu.txt ; fi + # Includes MacOS dependencies managed through Brew -install-macos: +install-macos: brew update brew upgrade brew install python3 @@ -244,12 +252,11 @@ install-macos: brew install npm npm ci npm install -g typescript webpack webpack-cli - make venv + make $(REQ) if [ -r requirements-macos.txt ]; then $(PIP_INSTALL) -r requirements-macos.txt ; fi # Includes Windows dependencies -install-windows: - make venv +install-windows: $(REQ) if [ -r requirements-windows.txt ]; then $(PIP_INSTALL) -r requirements-windows.txt ; fi ################################################################ diff --git a/docs/configuring_aws.rst b/docs/configuring_aws.rst new file mode 100644 index 00000000..af2f8728 --- /dev/null +++ b/docs/configuring_aws.rst @@ -0,0 +1,5 @@ +Type these commands to configure + +.. code-block :: + +sudo dnf install git emacs From 871fc429aabf73fc1e19fba023997eda53843f44 Mon Sep 17 00:00:00 2001 From: "Simson L. Garfinkel" Date: Sat, 28 Dec 2024 22:55:52 +0000 Subject: [PATCH 02/22] ignore backup files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index c9341231..8515ec51 100644 --- a/.gitignore +++ b/.gitignore @@ -184,3 +184,5 @@ node_modules coverage .aws-sam layer +[#]* +.[#]* From d4b9cbe6f386baf993637b0ed4f460ee420e79a0 Mon Sep 17 00:00:00 2001 From: "Simson L. Garfinkel" Date: Sun, 29 Dec 2024 18:52:21 +0000 Subject: [PATCH 03/22] can make and deploy on aws --- .gitignore | 1 + Makefile | 7 +++++++ deploy/Dockerfile | 16 ++++++++-------- deploy/lambda_handler.py | 2 +- template.yaml | 4 ++-- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 8515ec51..b5de73ec 100644 --- a/.gitignore +++ b/.gitignore @@ -186,3 +186,4 @@ coverage layer [#]* .[#]* +sam-installation diff --git a/Makefile b/Makefile index a1c38aba..1b0c1236 100644 --- a/Makefile +++ b/Makefile @@ -236,6 +236,13 @@ install-ubuntu: $(REQ) install-aws: echo install for AWS Linux, for making the lambda. echo note does not install ffmpeg currently + (cd $HOME; \ + wget https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip; \ + unzip aws-sam-cli-linux-x86_64.zip -d sam-installation; \ + sudo ./sam-installation/install ) + sudo dnf install -y docker + sudo systemctl enable docker + sudo systemctl start docker sudo dnf install -y python3.11 sudo dnf install -y nodejs npm npm ci diff --git a/deploy/Dockerfile b/deploy/Dockerfile index b581fda0..d96902f9 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,17 +1,17 @@ -FROM --platform=linux/amd64 public.ecr.aws/lambda/python:3.11 +# https://docs.aws.amazon.com/lambda/latest/dg/python-image.html -WORKDIR /app +FROM --platform=linux/amd64 public.ecr.aws/lambda/python:3.11 # Copy deploy/requirements.txt file for deployed app requirements -COPY requirements.txt . +COPY requirements.txt ${LAMBDA_TASK_ROOT} RUN python3 -m pip install -r requirements.txt -t . --no-warn-script-location # Copy the the app -COPY *.py ./deploy/ -COPY etc/ ./deploy/etc/ -COPY static/ ./deploy/static/ -COPY templates/ ./deploy/templates/ +COPY *.py ${LAMBDA_TASK_ROOT}/app/ +COPY etc/ ${LAMBDA_TASK_ROOT}/app/etc/ +COPY static/ ${LAMBDA_TASK_ROOT}/app/static/ +COPY templates/ ${LAMBDA_TASK_ROOT}/app/templates/ # CMD is used for local deployment: -CMD ["deploy.lambda_handler.lambda_handler"] +CMD ["app.lambda_handler.handler"] diff --git a/deploy/lambda_handler.py b/deploy/lambda_handler.py index 4039e6c2..4167c8dd 100644 --- a/deploy/lambda_handler.py +++ b/deploy/lambda_handler.py @@ -27,4 +27,4 @@ def dump_files(path="."): dump_files('/') from .bottle_app import app -lambda_handler = make_lambda_handler(app) +handler = make_lambda_handler(app) diff --git a/template.yaml b/template.yaml index 7025ae0e..8f9630f0 100644 --- a/template.yaml +++ b/template.yaml @@ -46,9 +46,9 @@ Resources: AWS_LAMBDA: "YES" DBREADER: "arn:aws:secretsmanager:us-east-1:376778049323:secret:planttracer_dbreader_dev-5LtJsU" DBWRITER: "arn:aws:secretsmanager:us-east-1:376778049323:secret:planttracer_dbwriter_dev-g7zJin" - DEBUG_DUMP_FILES: "YES" + DEBUG_DUMP_FILES: "NO" DEMO_MODE: "YES" - PLANTTRACER_CREDENTIALS: "etc/credentials-aws-dev.ini" + PLANTTRACER_CREDENTIALS: "app/etc/credentials-aws-dev.ini" PLANTTRACER_SMTP: "arn:aws:secretsmanager:us-east-1:376778049323:secret:planttracer_smtp-5TWQyf" PLANTTRACER_LOG_LEVEL: "WARNING" POWERTOOLS_SERVICE_NAME: FastAPI From 213dbcd363138ec5788a4d888c925021382e1d66 Mon Sep 17 00:00:00 2001 From: "Simson L. Garfinkel" Date: Sun, 29 Dec 2024 18:54:22 +0000 Subject: [PATCH 04/22] added policy needed for EC2 to deploy SAM --- docs/aws_sam_policy.json | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/aws_sam_policy.json diff --git a/docs/aws_sam_policy.json b/docs/aws_sam_policy.json new file mode 100644 index 00000000..2650e012 --- /dev/null +++ b/docs/aws_sam_policy.json @@ -0,0 +1,29 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "cloudformation:*", + "s3:*", + "lambda:*", + "apigateway:*", + "iam:PassRole", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:CompleteLayerUpload", + "ecr:DescribeImages", + "ecr:DescribeRepositories", + "ecr:GetDownloadUrlForLayer", + "ecr:InitiateLayerUpload", + "ecr:ListImages", + "ecr:PutImage", + "ecr:UploadLayerPart", + "logs:FilterLogEvents", + "logs:DescribeLogStreams" + ], + "Resource": "*" + } + ] +} From cb1d91d8f034228b26bf16801019c5597e18acae Mon Sep 17 00:00:00 2001 From: "Simson L. Garfinkel" Date: Sun, 29 Dec 2024 18:56:27 +0000 Subject: [PATCH 05/22] added expainations for install-aws --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 1b0c1236..31806b05 100644 --- a/Makefile +++ b/Makefile @@ -233,6 +233,8 @@ install-ubuntu: $(REQ) if [ -r requirements-ubuntu.txt ]; then $(PIP_INSTALL) -r requirements-ubuntu.txt ; fi # Install for AWS Linux for running SAM +# Start with: +# sudo dfn install git && git clone --recursive https://github.com/Plant-Tracer/webapp && (cd webapp; make aws-install) install-aws: echo install for AWS Linux, for making the lambda. echo note does not install ffmpeg currently From 05343ad5716919453e631c464e3138ae463a1276 Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Sun, 12 Jan 2025 19:20:29 -0500 Subject: [PATCH 06/22] udpate more --- README.md | 4 ++++ deploy/app/__init__.py | 1 + 2 files changed, 5 insertions(+) diff --git a/README.md b/README.md index 9e332fa1..36936769 100644 --- a/README.md +++ b/README.md @@ -70,3 +70,7 @@ Keep your Lambda functions optimized for performance and cost. # Lambda and S3 Lamda limits returns to 6MB and uploads to around 256K. So large uploads are done with presigned POST to S3 and large downloads by putting the data into S3 and having it pulled with a presigned URL. + + +colima start +docker ps -a diff --git a/deploy/app/__init__.py b/deploy/app/__init__.py index e69de29b..691bfc1b 100644 --- a/deploy/app/__init__.py +++ b/deploy/app/__init__.py @@ -0,0 +1 @@ +__version__='0.9.0' From 182dd29228ad5fcd0c58e88ea754a0953de13f53 Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Sun, 12 Jan 2025 19:40:57 -0500 Subject: [PATCH 07/22] pylint passes --- deploy/app/__init__.py | 3 +++ deploy/app/apikey.py | 13 ++++++++++++- deploy/app/constants.py | 1 + deploy/app/templates/base.html | 6 +++--- deploy/demo.py | 7 +++++-- deploy/lambda_handler.py | 20 ++++++++++---------- 6 files changed, 34 insertions(+), 16 deletions(-) diff --git a/deploy/app/__init__.py b/deploy/app/__init__.py index 691bfc1b..20679c7b 100644 --- a/deploy/app/__init__.py +++ b/deploy/app/__init__.py @@ -1 +1,4 @@ +""" +stuff +""" __version__='0.9.0' diff --git a/deploy/app/apikey.py b/deploy/app/apikey.py index 93b17c14..d4aab7a0 100644 --- a/deploy/app/apikey.py +++ b/deploy/app/apikey.py @@ -10,10 +10,14 @@ import functools import subprocess import json +from functools import lru_cache +import base64 +from os.path import join from flask import request from . import db +from .paths import ETC_DIR from .auth import get_dbreader,AuthError from .constants import C,__version__ @@ -115,10 +119,16 @@ def get_user_dict(): userdict = db.validate_api_key(api_key) if not userdict: - logging.info("api_key %s is invalid ipaddr=%s request.url=%s", api_key,request.remote_addr,request.url) + logging.info("api_key %s is invalid ipaddr=%s request.url=%s", + api_key,request.remote_addr,request.url) raise AuthError(f"api_key '{api_key}' is invalid") return userdict +@lru_cache(maxsize=1) +def favicon_base64(): + with open( join( ETC_DIR, C.FAVICON), 'rb') as f: + return base64.b64encode(f.read()).decode('utf-8') + def page_dict(title='', *, require_auth=False, lookup=True, logout=False,debug=False): """Returns a dictionary that can be used by post of the templates. :param: title - the title we should give the page @@ -169,6 +179,7 @@ def page_dict(title='', *, require_auth=False, lookup=True, logout=False,debug=F ret= fix_types({ C.API_BASE: api_base, C.STATIC_BASE: static_base, + 'favicon_base64':favicon_base64(), 'api_key': api_key, # the API key that is currently active 'user_id': user_id, # the user_id that is active 'user_name': user_name, # the user's name diff --git a/deploy/app/constants.py b/deploy/app/constants.py index 9d8a3985..3fa34310 100644 --- a/deploy/app/constants.py +++ b/deploy/app/constants.py @@ -23,6 +23,7 @@ class C: FFMPEG_PATH = 'FFMPEG_PATH' # Other + FAVICON = 'icon.png' API_BASE='API_BASE' STATIC_BASE='STATIC_BASE' TRACKING_COMPLETED='TRACKING COMPLETED' # keep case; it's used as a flag diff --git a/deploy/app/templates/base.html b/deploy/app/templates/base.html index fb0e1d2c..2cfd7d84 100644 --- a/deploy/app/templates/base.html +++ b/deploy/app/templates/base.html @@ -1,12 +1,12 @@ - {% block title %} {{title}} {% endblock %} + + - - + {% block title %} {{title}} {% endblock %} Date: Sun, 12 Jan 2025 20:02:12 -0500 Subject: [PATCH 08/22] pytest works again --- deploy/app/bottle_app.py | 2 +- deploy/app/db_object.py | 1 + deploy/app/etc/icon.png | Bin 0 -> 3037 bytes deploy/app/paths.py | 9 +++++++-- deploy/app/templates/base.html | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 deploy/app/etc/icon.png diff --git a/deploy/app/bottle_app.py b/deploy/app/bottle_app.py index a99eb0ce..a93aff95 100644 --- a/deploy/app/bottle_app.py +++ b/deploy/app/bottle_app.py @@ -43,7 +43,7 @@ def lambda_startup(): clogging.setup(level=os.environ.get('PLANTTRACER_LOG_LEVEL',logging.INFO)) fix_boto_log_level() - if C.PLANTTRACER_S3_BUCKET in os.environ: + if os.environ.get(C.PLANTTRACER_S3_BUCKET,None): db_object.S3_BUCKET = os.environ[C.PLANTTRACER_S3_BUCKET] else: config = auth.config() diff --git a/deploy/app/db_object.py b/deploy/app/db_object.py index 88205a17..45323c5f 100644 --- a/deploy/app/db_object.py +++ b/deploy/app/db_object.py @@ -95,6 +95,7 @@ def make_urn(*, object_name, scheme = None ): if scheme == C.SCHEME_S3 and S3_BUCKET is None: scheme = C.SCHEME_DB if scheme == C.SCHEME_S3: + assert len(S3_BUCKET)>0 netloc = S3_BUCKET elif scheme == C.SCHEME_DB: netloc = DB_TABLE diff --git a/deploy/app/etc/icon.png b/deploy/app/etc/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..86bffcbdf4f89a5074bcffe08214b62ab0248a94 GIT binary patch literal 3037 zcmZuz3p|r;8y|CA4w3Vl(x^FZbE?cJXLBZM6xy_DVb(UMRB{MmP01;xDW@c7iAZRX zkdi~Jm{M5e^g5L4+p4$TukX8`-}PMAeLesGb=}u>-@p6!oO5xq5eF%O004kE%GT12 ztKmEm5#qiN@UJCs6(7aT#vD-Avw4a;$-tpect=NoCf61L2=Yk-fIJBI0q`jS1i#w= zfF0jozic-?T>wAVFU1wF1Ohq!9s_?bN6J2w#Zk!$D>+|g9DqXWV(I0%Y45{$(` z!-GgX762KJ;F>`=Dn=zdD3C}&gqwiBBM@Ah=Z1k*zC)-1CSbIqi;6`s8K<%n3WvhM zrXUp+6(sox9^qzb{gcj}nShT{sU!pp#w#aGpH~P>*T~2S2G@h>>FID0I+O?^6%(#Q zq^SM~@=qK~9L0}JAW;dyL=|3K3^q7~Y61rH68#!K*BM0mm550BSrxZFSU83R(}lue z|Bpl^;D0agzgWC4KWN{p`jHIDZ4JVLjKffa$?n0yfu?_M5EAw$;`d6CypRY_HykB6 zFoai;sqSth?0=kp5}e5d9Je_?2tV=v_Wg-R|GxzPcK#$FVZ4U^tHD2Z@4J_~eWoC8 zE&m(^Q_zH!Mlbh>7ojZ8+>Z)&dvOAktz`C8ucR9^-OhSp?IK`rm2$fGj#s7Qb?}he zI@qzYUowWS9wWOByzZO+Nnk7EI=FZelI$z9SOI&+05fCM)nx+u3cHQ#S7w!?nyZah zq{4!q&(aW>Fn|B2Z+7|jTrtw$1=d1;XunuHioy$eBn}3$Lse1<31s?{PpqeUX~q~E zva<`l7JK=2`OSq7jC|Qi60zvZedcg*D@PdltZ>x)GG2o?E3#x~B(R*nTHHNi)MPi@ z;QO&n_HwGh%Nqm>J5J6>!4aGNuhhorNz7~RhvIwu@-Gbs&Soix)I4Sgv$~b}I%~eF zR(Z{|>XMj)cP@Ra2sk=Wt)V)meN1=+!Ca)<5Bq3t50Uj}=;0F}_Z8*m%XYo0jzZ+- zD<214eEc|}Y&?A#lZ6EEy~;JV8CC(iJT>KaC{1sXuD;|W=j;9SmMGrO-Y=Wno>ksys-(cLEfkIbV~WNM7_9MLJ;HN zAxd9Sw!KexPkuc`Ei*tE|SY!b-6%&LFb1 z*{VqgBGwft?iqFNy<_5m+|;$D2#u~QyNq#p@#SSDLoczC?{^+fXPrM;=bcjT-lFI! ze1@Y^qujd4@B%*Xe+~lnxv~sn%@+7_R28w5tLpN!K-QExowML zxqjifLK7Neio#_Uh5l^$Sc8=<2B$%X3AER7X6<=R74)FY!H;( zp~Dx2TGOCK8Hl&(!pa|^TQUK_+MegQ86t4&SKwH=rG-?tn(7DU*_r5=2kmS3rI}Nq zg_MsT-Uf2UuaD`82tf!Tf8R*ueDuv~5l*I1))7>x3>}Kv8yjSc3G)TQI zAnk2zd|m;Z)@~Mr#?_v%-!120e0dLlbk`WmmXR#jkVzhjS5D;%gugodGRu)&nb+x4 znG?U^p|VWBxEPIUxIVcc9(i{{b2DQ-4yR3by4&rnS}*^(JK?+x)dADg)WS0QcCT$C zq+(t~psiDEPUx+gfb)uH5~NqQRQ`ju1+?u68RT49@7-np$Y-mbfdSbMZo!_5cqiBG z7^@Rv5uN8od~phZnqtNR3We?EJC{8&4G*TXFo|p$!m8||M8c-5w35cntiw3CgKFf& z{FD5Xw7bL$gH%-wKlzNzm&dXy5R&;Zd7xW&gbx>JAK$cQO~vO}a5D+9;5`c<8Z;Xr%qRdnVA?sw2QBjoPqb^z|8m*ior)rPj51 z=ep|>-I00veJ7OHj^``n+Ab?1or^b21-c--Pu8 zr1Cuve}36swt_1^rvL!f+C-K+BSh+2(R>`z9@oZh5F-O5te!}i4bM;#%tEzTC3~p1 zLKGCOZ%(t}A(c$?I*|z>w2S8kjHwW>$b?Y{`Z9B^Q*(-roLn)g*<` zHI*HE&A)=_@I)-Wb}T5Nf2(1g&U7>D=GE6hwm0Y{ zfz!4Y@*znk`6si0NsmQ6d(!M|+h53iJ|jt@`3=!W%*0NiC!xsk(2w$;)XE#KqzTux zViu22B9b}YpA|#5svjTlb-Tui1FcmBnXGi4M?r2zqE<^e0^h9s;iGKJ0W67oRW|cA XW)bhpX`ia*{n4VVoGi=C56AomGI$!T literal 0 HcmV?d00001 diff --git a/deploy/app/paths.py b/deploy/app/paths.py index a6109c88..3ce7b9f5 100644 --- a/deploy/app/paths.py +++ b/deploy/app/paths.py @@ -38,7 +38,12 @@ def ffmpeg_path(): if C.FFMPEG_PATH in os.environ: - return os.environ[C.FFMPEG_PATH] + pth = os.environ[C.FFMPEG_PATH] + if os.path.exists(path): + return pth + pth = shutil.which('ffmpeg') + if pth: + return pth if os.path.exists(AWS_LAMBDA_LINUX_STATIC_FFMPEG): return AWS_LAMBDA_LINUX_STATIC_FFMPEG - return shutil.which('ffmpeg') + raise FileNotFoundError("ffmpeg") diff --git a/deploy/app/templates/base.html b/deploy/app/templates/base.html index 2cfd7d84..226ef5dd 100644 --- a/deploy/app/templates/base.html +++ b/deploy/app/templates/base.html @@ -1,7 +1,7 @@ - + From 90464d73d64fbbd4876a86e37e1c8b356ed6497a Mon Sep 17 00:00:00 2001 From: "Simson L. Garfinkel" Date: Wed, 15 Jan 2025 20:24:16 -0500 Subject: [PATCH 09/22] updated --- deploy/app/auth.py | 6 ++---- deploy/app/paths.py | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/deploy/app/auth.py b/deploy/app/auth.py index 263f24ed..1987ffd4 100644 --- a/deploy/app/auth.py +++ b/deploy/app/auth.py @@ -13,6 +13,7 @@ from . import dbfile from .constants import C +from .path import DEFAULT_CREDENTIALS_FILE COOKIE_MAXAGE = 60*60*24*180 SMTP_ATTRIBS = ['SMTP_USERNAME', 'SMTP_PASSWORD', 'SMTP_PORT', 'SMTP_HOST'] @@ -23,10 +24,7 @@ def credentials_file(): - try: - name = os.environ[ C.PLANTTRACER_CREDENTIALS ] - except KeyError as e: - raise RuntimeError(f"Environment variable {C.PLANTTRACER_CREDENTIALS} must be defined") from e + name = os.environ.get(C.PLANTTRACER_CREDENTIALS, DEFAULT_CREDENTIALS_FILE) if not os.path.exists(name): logging.error("Cannot find %s (PLANTTRACER_CREDENTIALS=%s)",os.path.abspath(name),name) raise FileNotFoundError(name) diff --git a/deploy/app/paths.py b/deploy/app/paths.py index 3ce7b9f5..9874b602 100644 --- a/deploy/app/paths.py +++ b/deploy/app/paths.py @@ -25,6 +25,7 @@ SCHEMA_FILE = join(ETC_DIR, 'schema.sql') SCHEMA_TEMPLATE = join(ETC_DIR, 'schema_{schema}.sql') +DEFAULT_CREDENTIALS_FILE = join(ETC_DIR, 'credentials.ini') SCHEMA0_FILE = SCHEMA_TEMPLATE.format(schema=0) SCHEMA1_FILE = SCHEMA_TEMPLATE.format(schema=1) From 80113ded350753a037d6413fdb9e6ab6326d0d1e Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Thu, 16 Jan 2025 02:04:13 +0000 Subject: [PATCH 10/22] works on micro vm --- Makefile | 17 +++++------------ deploy/app/auth.py | 8 -------- deploy/lambda_handler.py | 1 - 3 files changed, 5 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index f41f3c2b..b93a7941 100644 --- a/Makefile +++ b/Makefile @@ -10,11 +10,9 @@ JS_FILES := $(TS_FILES:.ts=.js) ################################################################ # Create the virtual enviornment for testing and CI/CD -ACTIVATE = . venv/bin/activate REQ = venv/pyvenv.cfg -PY=python3.11 -PYTHON=$(ACTIVATE) ; $(PY) -PIP_INSTALL=$(PYTHON) -m pip install --no-warn-script-location +PYTHON=venv/bin/python +PIP_INSTALL=venv/bin/pip install --no-warn-script-location ETC=etc APP_ETC=deploy/app/etc DBMAINT=-m deploy.app.dbmaint @@ -23,7 +21,7 @@ DBMAINT=-m deploy.app.dbmaint venv: @echo install venv for the development environment - $(PY) -m venv venv + python3 -m venv venv $(PYTHON) -m pip install --upgrade pip if [ -r requirements.txt ]; then $(PIP_INSTALL) -r requirements.txt ; fi if [ -r deploy/requirements.txt ]; then $(PIP_INSTALL) -r deploy/requirements.txt ; fi @@ -66,10 +64,10 @@ check: PYLINT_OPTS:=--output-format=parseable --rcfile .pylintrc --fail-under=$(PYLINT_THRESHOLD) --verbose pylint: $(REQ) - $(ACTIVATE) ; $(PY) -m pylint $(PYLINT_OPTS) deploy + $(PYTHON) -m pylint $(PYLINT_OPTS) deploy pylint-tests: $(REQ) - $(ACTIVATE) ; $(PY) -m pylint $(PYLINT_OPTS) --init-hook="import sys;sys.path.append('tests');import conftest" tests + $(PYTHON) -m pylint $(PYLINT_OPTS) --init-hook="import sys;sys.path.append('tests');import conftest" tests mypy: mypy --show-error-codes --pretty --ignore-missing-imports --strict . @@ -204,13 +202,8 @@ jscoverage: ################################################################ -<<<<<<< HEAD # Installations are used by the CI pipeline and by developers # $(REQ) gets made by the virtual environment installer, but you need to have python installed first. -======= -# Installations are used by the CI pipeline: -# Use actions_test unless a local db is already defined ->>>>>>> main PLANTTRACER_LOCALDB_NAME ?= actions_test create_localdb: diff --git a/deploy/app/auth.py b/deploy/app/auth.py index 1cd75ecb..06890c42 100644 --- a/deploy/app/auth.py +++ b/deploy/app/auth.py @@ -13,11 +13,7 @@ from . import dbfile from .constants import C -<<<<<<< HEAD -from .path import DEFAULT_CREDENTIALS_FILE -======= from .paths import DEFAULT_CREDENTIALS_FILE ->>>>>>> dev-aws COOKIE_MAXAGE = 60*60*24*180 SMTP_ATTRIBS = ['SMTP_USERNAME', 'SMTP_PASSWORD', 'SMTP_PORT', 'SMTP_HOST'] @@ -28,11 +24,7 @@ def credentials_file(): -<<<<<<< HEAD name = os.environ.get(C.PLANTTRACER_CREDENTIALS, DEFAULT_CREDENTIALS_FILE) -======= - name = os.environ.get(C.PLANTTRACER_CREDENTIALS,DEFAULT_CREDENTIALS_FILE) ->>>>>>> dev-aws if not os.path.exists(name): logging.error("Cannot find %s (PLANTTRACER_CREDENTIALS=%s)",os.path.abspath(name),name) raise FileNotFoundError(name) diff --git a/deploy/lambda_handler.py b/deploy/lambda_handler.py index 0cee185c..7207fca0 100644 --- a/deploy/lambda_handler.py +++ b/deploy/lambda_handler.py @@ -35,4 +35,3 @@ def lambda_handler(event, context): # pylint: disable=unused-argument "body": "error:\n" + f.read() } lambda_app = lambda_handler - From e144e3c91e4fbd4d4150394735d4c03726449ae3 Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Thu, 16 Jan 2025 12:59:29 +0000 Subject: [PATCH 11/22] added aws srvices for microvm --- docs/mv1.rst | 7 +++++++ etc/planttracer-debug.service | 20 ++++++++++++++++++++ etc/planttracer.service | 19 +++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 docs/mv1.rst create mode 100644 etc/planttracer-debug.service create mode 100644 etc/planttracer.service diff --git a/docs/mv1.rst b/docs/mv1.rst new file mode 100644 index 00000000..023431a2 --- /dev/null +++ b/docs/mv1.rst @@ -0,0 +1,7 @@ +Configuration on mv1: + +Host | gunicorn port | app dir | app name +mv1.planttracer.com | 8000 | /home/ec2-user/webapp/deploy/ | app.app +app.planttracer.com | 8010 | /home/ec2-user/webapp/deploy/ | app.app +demo1.planttracer.com | 8020 | /home/ec2-user/webapp/deploy/ | app.app +dev-slg.planttracer.com | 8030| /home/ec2-user/slg-dev/deploy/ | app.app diff --git a/etc/planttracer-debug.service b/etc/planttracer-debug.service new file mode 100644 index 00000000..553be7b9 --- /dev/null +++ b/etc/planttracer-debug.service @@ -0,0 +1,20 @@ +### /etc/systemd/system/gunicorn.service ### +[Unit] +Description=Gunicorn instance to serve Flask application +After=network.target + +[Service] +User=ec2-user +Group=ec2-user +WorkingDirectory=/home/ec2-user/webapp +Environment="PATH=/home/ec2-user/webapp/venv/bin" +Environment="PLANTTRACER_CREDENTIALS=/home/ec2-user/webapp/deploy/etc/planttracer.ini" +Environment="DEMO_MODE=YES" +Environment="PATH=/home/ec2-user/webapp/venv/bin" +ExecStart=/home/ec2-user/webapp/venv/bin/gunicorn -w 4 -b 127.0.0.1:8010 \ + --access-logfile /home/ec2-user/gunicorn_access.log \ + --error-logfile /home/ec2-user/gunicorn_error.log\ + --reload deploy.app.bottle_app:app + +[Install] +WantedBy=multi-user.target diff --git a/etc/planttracer.service b/etc/planttracer.service new file mode 100644 index 00000000..82acb239 --- /dev/null +++ b/etc/planttracer.service @@ -0,0 +1,19 @@ +### /etc/systemd/system/gunicorn.service ### +[Unit] +Description=Gunicorn instance to serve Flask application +After=network.target + +[Service] +User=ec2-user +Group=ec2-user +WorkingDirectory=/home/ec2-user/webapp +Environment="PATH=/home/ec2-user/webapp/venv/bin" +Environment="PLANTTRACER_CREDENTIALS=/home/ec2-user/webapp/deploy/etc/planttracer.ini" +Environment="PATH=/home/ec2-user/webapp/venv/bin" +ExecStart=/home/ec2-user/webapp/venv/bin/gunicorn -w 4 -b 127.0.0.1:8000 \ + --access-logfile /home/ec2-user/gunicorn_access.log \ + --error-logfile /home/ec2-user/gunicorn_error.log\ + --reload deploy.app.bottle_app:app + +[Install] +WantedBy=multi-user.target From ebaaa338759878d25cb93cd41a4529ac90826a61 Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Thu, 16 Jan 2025 22:50:43 +0000 Subject: [PATCH 12/22] generate all as set --- etc/planttracer-template.conf | 10 +++++++ ...g.service => planttracer-template.service} | 12 ++++---- etc/planttracer.service | 19 ------------ etc/setup_services.py | 29 +++++++++++++++++++ 4 files changed, 45 insertions(+), 25 deletions(-) create mode 100644 etc/planttracer-template.conf rename etc/{planttracer-debug.service => planttracer-template.service} (59%) delete mode 100644 etc/planttracer.service create mode 100644 etc/setup_services.py diff --git a/etc/planttracer-template.conf b/etc/planttracer-template.conf new file mode 100644 index 00000000..ebacca58 --- /dev/null +++ b/etc/planttracer-template.conf @@ -0,0 +1,10 @@ + + ServerName {name}.planttracer.com + DocumentRoot /home/www + ProxyPass "/" "http://localhost:{port}/" + ProxyPassReverse "/" "http://localhost:{port}/" + + RewriteEngine on + RewriteCond %{SERVER_NAME} ={name}.planttracer.com + RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [END,NE,R=permanent] + diff --git a/etc/planttracer-debug.service b/etc/planttracer-template.service similarity index 59% rename from etc/planttracer-debug.service rename to etc/planttracer-template.service index 553be7b9..a7eba7d8 100644 --- a/etc/planttracer-debug.service +++ b/etc/planttracer-template.service @@ -1,6 +1,6 @@ ### /etc/systemd/system/gunicorn.service ### [Unit] -Description=Gunicorn instance to serve Flask application +Description=Planttracer {name} to serve Flask application After=network.target [Service] @@ -8,12 +8,12 @@ User=ec2-user Group=ec2-user WorkingDirectory=/home/ec2-user/webapp Environment="PATH=/home/ec2-user/webapp/venv/bin" -Environment="PLANTTRACER_CREDENTIALS=/home/ec2-user/webapp/deploy/etc/planttracer.ini" -Environment="DEMO_MODE=YES" +Environment="PLANTTRACER_CREDENTIALS=/home/ec2-user/webapp/deploy/etc/{credentials}.ini" +Environment="DEMO_MODE=/home/ec2-user/webapp/deploy/etc/{demo_mode}.ini" Environment="PATH=/home/ec2-user/webapp/venv/bin" -ExecStart=/home/ec2-user/webapp/venv/bin/gunicorn -w 4 -b 127.0.0.1:8010 \ - --access-logfile /home/ec2-user/gunicorn_access.log \ - --error-logfile /home/ec2-user/gunicorn_error.log\ +ExecStart=/home/ec2-user/webapp/venv/bin/gunicorn -w 4 -b 127.0.0.1:{port} \ + --access-logfile /home/ec2-user/logs/planttracer-{name}.log \ + --error-logfile /home/ec2-user/logs/planttracer-{name}-error.log\ --reload deploy.app.bottle_app:app [Install] diff --git a/etc/planttracer.service b/etc/planttracer.service deleted file mode 100644 index 82acb239..00000000 --- a/etc/planttracer.service +++ /dev/null @@ -1,19 +0,0 @@ -### /etc/systemd/system/gunicorn.service ### -[Unit] -Description=Gunicorn instance to serve Flask application -After=network.target - -[Service] -User=ec2-user -Group=ec2-user -WorkingDirectory=/home/ec2-user/webapp -Environment="PATH=/home/ec2-user/webapp/venv/bin" -Environment="PLANTTRACER_CREDENTIALS=/home/ec2-user/webapp/deploy/etc/planttracer.ini" -Environment="PATH=/home/ec2-user/webapp/venv/bin" -ExecStart=/home/ec2-user/webapp/venv/bin/gunicorn -w 4 -b 127.0.0.1:8000 \ - --access-logfile /home/ec2-user/gunicorn_access.log \ - --error-logfile /home/ec2-user/gunicorn_error.log\ - --reload deploy.app.bottle_app:app - -[Install] -WantedBy=multi-user.target diff --git a/etc/setup_services.py b/etc/setup_services.py new file mode 100644 index 00000000..d53cb9de --- /dev/null +++ b/etc/setup_services.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +# Define the instances we are creating. + +def readfile(fname): + with open(fname,'r') as f: + return f.read() + +def writefile(fname,name,content): + with open(fname.format(name=name), 'w') as f: + f.write(content) + +HTTPD_TEMPLATE=readfile('planttracer-template.conf') +SYSTEMD_TEMPLATE=readfile('planttracer-template.service') + +HTTPD_DIR = +SYSTEMD_DIR = '/etc/systemd/system' + + +DEFS = """mv1,8000,no +app,8010,no +demo1,8020,no +dev-slg,8030,no""" + + +for def in DEFS.split('\n'): + for (name,port,demo) in def.split(','): + writefile('/etc/httpd/conf.d/planttracer-{name}.conf', name, content.format(name=name, port=port, demo=demo)) + writefile('/etc/systemd/system/planttracer-{name}.service', name, content.format(name=name, port=port, demo=demo)) From f3857b0d7e0099dc02164dbfceab05b8a6f0ae96 Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Thu, 16 Jan 2025 22:51:49 +0000 Subject: [PATCH 13/22] reload system --- etc/setup_services.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/setup_services.py b/etc/setup_services.py index d53cb9de..e3c3d038 100644 --- a/etc/setup_services.py +++ b/etc/setup_services.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +import os + # Define the instances we are creating. def readfile(fname): @@ -27,3 +29,5 @@ def writefile(fname,name,content): for (name,port,demo) in def.split(','): writefile('/etc/httpd/conf.d/planttracer-{name}.conf', name, content.format(name=name, port=port, demo=demo)) writefile('/etc/systemd/system/planttracer-{name}.service', name, content.format(name=name, port=port, demo=demo)) + +os.system("sudo systemctl daemon-reload") From 43a9725ecc32b053e7fc7efcd3ea6f73d1ac685b Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Fri, 17 Jan 2025 02:22:03 +0000 Subject: [PATCH 14/22] added base to template --- etc/planttracer-template.conf | 4 ++-- etc/planttracer-template.service | 17 ++++++++--------- etc/setup_services.py | 28 ++++++++++++++-------------- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/etc/planttracer-template.conf b/etc/planttracer-template.conf index ebacca58..647a80f0 100644 --- a/etc/planttracer-template.conf +++ b/etc/planttracer-template.conf @@ -5,6 +5,6 @@ ProxyPassReverse "/" "http://localhost:{port}/" RewriteEngine on - RewriteCond %{SERVER_NAME} ={name}.planttracer.com - RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [END,NE,R=permanent] + RewriteCond %{{SERVER_NAME}} ={name}.planttracer.com + RewriteRule ^ https://%{{SERVER_NAME}}%{{REQUEST_URI}} [END,NE,R=permanent] diff --git a/etc/planttracer-template.service b/etc/planttracer-template.service index a7eba7d8..74a35a43 100644 --- a/etc/planttracer-template.service +++ b/etc/planttracer-template.service @@ -6,15 +6,14 @@ After=network.target [Service] User=ec2-user Group=ec2-user -WorkingDirectory=/home/ec2-user/webapp -Environment="PATH=/home/ec2-user/webapp/venv/bin" -Environment="PLANTTRACER_CREDENTIALS=/home/ec2-user/webapp/deploy/etc/{credentials}.ini" -Environment="DEMO_MODE=/home/ec2-user/webapp/deploy/etc/{demo_mode}.ini" -Environment="PATH=/home/ec2-user/webapp/venv/bin" -ExecStart=/home/ec2-user/webapp/venv/bin/gunicorn -w 4 -b 127.0.0.1:{port} \ - --access-logfile /home/ec2-user/logs/planttracer-{name}.log \ - --error-logfile /home/ec2-user/logs/planttracer-{name}-error.log\ - --reload deploy.app.bottle_app:app +WorkingDirectory={base} +Environment="PATH={base}/venv/bin" +Environment="PLANTTRACER_CREDENTIALS={base}/deploy/etc/credentials-{name}.ini" +Environment="DEMO_MODE={demo}" +ExecStart={base}/venv/bin/gunicorn -w 4 -b 127.0.0.1:{port} \ + --access-logfile /home/ec2-user/logs/planttracer-{name}.log \ + --error-logfile /home/ec2-user/logs/planttracer-{name}-error.log\ + --reload deploy.app.bottle_app:app [Install] WantedBy=multi-user.target diff --git a/etc/setup_services.py b/etc/setup_services.py index e3c3d038..078a712c 100644 --- a/etc/setup_services.py +++ b/etc/setup_services.py @@ -15,19 +15,19 @@ def writefile(fname,name,content): HTTPD_TEMPLATE=readfile('planttracer-template.conf') SYSTEMD_TEMPLATE=readfile('planttracer-template.service') -HTTPD_DIR = -SYSTEMD_DIR = '/etc/systemd/system' - - -DEFS = """mv1,8000,no -app,8010,no -demo1,8020,no -dev-slg,8030,no""" - - -for def in DEFS.split('\n'): - for (name,port,demo) in def.split(','): - writefile('/etc/httpd/conf.d/planttracer-{name}.conf', name, content.format(name=name, port=port, demo=demo)) - writefile('/etc/systemd/system/planttracer-{name}.service', name, content.format(name=name, port=port, demo=demo)) +# name,port,demo,base +DEFS = """mv1,8000,no,/home/ec2-user/webapp +app,8010,no,/home/ec2-user/webapp +demo1,8020,no,/home/ec2-user/webapp +dev-slg,8030,no,/home/ec2-user/dev-slg""" + + +for d in DEFS.split('\n'): + print("d=",d) + (name,port,demo,base) = d.split(',') + writefile('/etc/httpd/conf.d/planttracer-{name}.conf', name, + HTTPD_TEMPLATE.format(name=name, port=port, demo=demo,base=base)) + writefile('/etc/systemd/system/planttracer-{name}.service', name, + SYSTEMD_TEMPLATE.format(name=name, port=port, demo=demo,base=base)) os.system("sudo systemctl daemon-reload") From d8f920f5b6a1ebfbde4cce5cae482f85052bb6cc Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Fri, 17 Jan 2025 03:19:49 +0000 Subject: [PATCH 15/22] update --- etc/planttracer-template.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/planttracer-template.service b/etc/planttracer-template.service index 74a35a43..b87486e7 100644 --- a/etc/planttracer-template.service +++ b/etc/planttracer-template.service @@ -10,7 +10,7 @@ WorkingDirectory={base} Environment="PATH={base}/venv/bin" Environment="PLANTTRACER_CREDENTIALS={base}/deploy/etc/credentials-{name}.ini" Environment="DEMO_MODE={demo}" -ExecStart={base}/venv/bin/gunicorn -w 4 -b 127.0.0.1:{port} \ +ExecStart={base}/venv/bin/gunicorn -w 2 -b 127.0.0.1:{port} \ --access-logfile /home/ec2-user/logs/planttracer-{name}.log \ --error-logfile /home/ec2-user/logs/planttracer-{name}-error.log\ --reload deploy.app.bottle_app:app From c99b7e23f985d794d7c7a26f8d22d3162da5686c Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Fri, 17 Jan 2025 04:02:21 +0000 Subject: [PATCH 16/22] clarified ini directories --- Makefile | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index b93a7941..2cb18e13 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,8 @@ JS_FILES := $(TS_FILES:.ts=.js) REQ = venv/pyvenv.cfg PYTHON=venv/bin/python PIP_INSTALL=venv/bin/pip install --no-warn-script-location -ETC=etc -APP_ETC=deploy/app/etc +ROOT_ETC=etc +DEPLOY_ETC=deploy/etc DBMAINT=-m deploy.app.dbmaint # Note: PLANTTRACER_CREDENTIALS must be set @@ -136,10 +136,10 @@ pytest-quiet: $(PYTHON) -m pytest --log-cli-level=ERROR test-schema-upgrade: - $(PYTHON) $(DBMAINT) --rootconfig $(ETC)/mysql-root-localhost.ini --dropdb test_db1 || echo database does not exist - $(PYTHON) $(DBMAINT) --rootconfig $(ETC)/mysql-root-localhost.ini --createdb test_db1 --schema $(ETC)/schema_0.sql - $(PYTHON) $(DBMAINT) --rootconfig $(ETC)/mysql-root-localhost.ini --upgradedb test_db1 - $(PYTHON) $(DBMAINT) --rootconfig $(ETC)/mysql-root-localhost.ini --dropdb test_db1 + $(PYTHON) $(DBMAINT) --rootconfig $(ROOT_ETC)/mysql-root-localhost.ini --dropdb test_db1 || echo database does not exist + $(PYTHON) $(DBMAINT) --rootconfig $(ROOT_ETC)/mysql-root-localhost.ini --createdb test_db1 --schema $(ROOT_ETC)/schema_0.sql + $(PYTHON) $(DBMAINT) --rootconfig $(ROOT_ETC)/mysql-root-localhost.ini --upgradedb test_db1 + $(PYTHON) $(DBMAINT) --rootconfig $(ROOT_ETC)/mysql-root-localhost.ini --dropdb test_db1 pytest-coverage: $(REQ) $(PIP_INSTALL) codecov pytest pytest_cov @@ -162,25 +162,25 @@ debug: debug-local: @echo run bottle locally in debug mode, storing new data in database - PLANTTRACER_CREDENTIALS=$(ETC)/credentials-localhost.ini $(DEBUG) --storelocal + PLANTTRACER_CREDENTIALS=$(APP_ETC)/credentials-localhost.ini $(DEBUG) --storelocal debug-single: @echo run bottle locally in debug mode single-threaded - PLANTTRACER_CREDENTIALS=$(ETC)/credentials-localhost.ini $(DEBUG) + PLANTTRACER_CREDENTIALS=$(APP_ETC)/credentials-localhost.ini $(DEBUG) debug-multi: @echo run bottle locally in debug mode multi-threaded - PLANTTRACER_CREDENTIALS=$(ETC)/credentials-localhost.ini $(DEBUG) --multi + PLANTTRACER_CREDENTIALS=$(APP_ETC)/credentials-localhost.ini $(DEBUG) --multi debug-dev: @echo run bottle locally in debug mode, storing new data in S3, with the dev.planttracer.com database @echo for debugging Python and Javascript with remote database - PLANTTRACER_CREDENTIALS=$(ETC)/credentials-aws-dev.ini $(DEBUG) + PLANTTRACER_CREDENTIALS=$(APP_ETC)/credentials-aws-dev.ini $(DEBUG) debug-dev-api: @echo Debug local JavaScript with remote server. @echo run bottle locally in debug mode, storing new data in S3, with the dev.planttracer.com database and API calls - PLANTTRACER_CREDENTIALS=$(ETC)/credentials-aws-dev.ini PLANTTRACER_API_BASE=https://dev.planttracer.com/ $(DEBUG) + PLANTTRACER_CREDENTIALS=$(APP_ETC)/credentials-aws-dev.ini PLANTTRACER_API_BASE=https://dev.planttracer.com/ $(DEBUG) tracker-debug: /bin/rm -f outfile.mp4 @@ -207,22 +207,22 @@ jscoverage: PLANTTRACER_LOCALDB_NAME ?= actions_test create_localdb: - @echo Creating local database, exercise the upgrade code and write credentials to $(PLANTTRACER_CREDENTIALS) using $(ETC)/github_actions_mysql_rootconfig.ini + @echo Creating local database, exercise the upgrade code and write credentials to $(PLANTTRACER_CREDENTIALS) using $(ROOT_ETC)/github_actions_mysql_rootconfig.ini @echo $(PLANTTRACER_CREDENTIALS) will be used automatically by other tests - mkdir -p $(ETC) - ls -l $(ETC) - $(PYTHON) $(DBMAINT) --create_client=$$MYSQL_ROOT_PASSWORD --writeconfig $(ETC)/github_actions_mysql_rootconfig.ini - ls -l $(ETC) - $(PYTHON) $(DBMAINT) --rootconfig $(ETC)/github_actions_mysql_rootconfig.ini \ + mkdir -p $(ROOT_ETC) + ls -l $(ROOT_ETC) + $(PYTHON) $(DBMAINT) --create_client=$$MYSQL_ROOT_PASSWORD --writeconfig $(ROOT_ETC)/github_actions_mysql_rootconfig.ini + ls -l $(ROOT_ETC) + $(PYTHON) $(DBMAINT) --rootconfig $(ROOT_ETC)/github_actions_mysql_rootconfig.ini \ --createdb $(PLANTTRACER_LOCALDB_NAME) \ - --schema $(APP_ETC)/schema_0.sql \ + --schema $(DEPLOY_ETC)/schema_0.sql \ --writeconfig $(PLANTTRACER_CREDENTIALS) $(PYTHON) $(DBMAINT) --upgradedb --loglevel DEBUG $(PYTHON) -m pytest -x --log-cli-level=DEBUG tests/dbreader_test.py remove_localdb: - @echo Removing local database using $(ETC)/github_actions_mysql_rootconfig.ini - $(PYTHON) $(DBMAINT) --rootconfig $(ETC)/github_actions_mysql_rootconfig.ini --dropdb $(PLANTTRACER_LOCALDB_NAME) + @echo Removing local database using $(ROOT_ETC)/github_actions_mysql_rootconfig.ini + $(PYTHON) $(DBMAINT) --rootconfig $(ROOT_ETC)/github_actions_mysql_rootconfig.ini --dropdb $(PLANTTRACER_LOCALDB_NAME) install-chromium-browser-ubuntu: $(REQ) @@ -244,7 +244,7 @@ install-ubuntu: $(REQ) # Install for AWS Linux for running SAM # Start with: # sudo dfn install git && git clone --recursive https://github.com/Plant-Tracer/webapp && (cd webapp; make aws-install) -install-aws: +install-aws: echo install for AWS Linux, for making the lambda. echo note does not install ffmpeg currently (cd $HOME; \ @@ -261,7 +261,7 @@ install-aws: if [ -r requirements-aws.txt ]; then $(PIP_INSTALL) -r requirements-ubuntu.txt ; fi # Includes MacOS dependencies managed through Brew -install-macos: +install-macos: brew update brew upgrade brew install python3 From 7b771d3c58b7c68ecb3d52befa0f924c620c8d1a Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Sat, 18 Jan 2025 22:10:53 +0000 Subject: [PATCH 17/22] logging works --- dbutil.py | 233 +++++++++++++++++++++++++++++++ deploy/app/apikey.py | 12 +- deploy/app/auth.py | 1 - deploy/app/bottle_api.py | 5 +- deploy/app/bottle_app.py | 71 +++++++--- deploy/app/constants.py | 3 +- deploy/app/dbmaint.py | 231 +----------------------------- deploy/app/paths.py | 2 +- etc/planttracer-template.service | 8 +- 9 files changed, 311 insertions(+), 255 deletions(-) create mode 100644 dbutil.py diff --git a/dbutil.py b/dbutil.py new file mode 100644 index 00000000..10bacee3 --- /dev/null +++ b/dbutil.py @@ -0,0 +1,233 @@ +import sys +import os +import configparser + +from deploy.app import clogging +from deploy.app import dbmaint +from deploy.app import db +from deploy.app import dbfile + + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser(description="Database Maintenance Program. The database to act upon is specified in the ini file specified by the PLANTTRACER_CREDENTIALS environment variable, in the sections for [dbreader] and [dbwriter]", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + required = parser.add_argument_group('required arguments') + required.add_argument( + "--rootconfig", + help='Specify config file with MySQL database root credentials in [client] section. ' + 'Format is the same as the mysql --defaults-extra-file= argument') + parser.add_argument("--sendlink", help="Send link to the given email address, registering it if necessary.") + parser.add_argument('--planttracer_endpoint',help='https:// endpoint where planttracer app can be found') + parser.add_argument("--createdb", + help='Create a new database and a dbreader and dbwriter user. Database must not exist. ' + 'Requires that the variables MYSQL_DATABASE and MYSQL_HOST are set, and that MYSQL_PASSWORD and MYSQL_USER ' + 'are set with a MySQL username that can issue the "CREATE DATABASE" command.' + 'Outputs setenv for DBREADER and DBWRITER') + parser.add_argument("--upgradedb", help='Upgrade a database schema',action='store_true') + parser.add_argument("--dropdb", help='Drop an existing database.') + parser.add_argument("--readconfig", help="Specify the config.ini file to read") + parser.add_argument("--writeconfig", help="Specify the config.ini file to write.") + parser.add_argument('--purge_test_data', help='Remove the test data from the database', action='store_true') + parser.add_argument('--purge_all_movies', help='Remove all of the movies from the database', action='store_true') + parser.add_argument("--purge_movie",help="Remove the movie and all of its associated data from the database",type=int) + parser.add_argument("--create_client",help="Create a [client] section with a root username and the specified password") + parser.add_argument("--create_course",help="Create a course and register --admin_email --admin_name as the administrator") + parser.add_argument('--demo_email',help='If create_course is specified, also create a demo user with this email and upload demo movies ') + parser.add_argument("--admin_email",help="Specify the email address of the course administrator") + parser.add_argument("--admin_name",help="Specify the name of the course administrator") + parser.add_argument("--max_enrollment",help="Max enrollment for course",type=int,default=20) + parser.add_argument("--report",help="Print a report of the database",action='store_true') + parser.add_argument("--freshen",help="Non-destructive cleans up the movie metadata for all movies.",action='store_true') + parser.add_argument("--clean",help="Destructive cleans up the movie metadata for all movies.",action='store_true') + parser.add_argument("--schema", help="Specify schema file to use", default=dbmaint.SCHEMA_FILE) + parser.add_argument("--dump", help="Backup all objects as JSON files and movie files to new directory called DUMP. ") + parser.add_argument("--add_admin", help="Add --admin_email user as a course admin to the course specified by --course_id, --course_name, or --course_name", action='store_true') + parser.add_argument("--course_id", help="integer course id", type=int) + parser.add_argument("--course_key", help="integer course id") + parser.add_argument("--course_name", help="integer course id") + parser.add_argument("--remove_admin", help="Remove the --admin_email user as a course admin from the course specified by --course_id, --course_name, or --course_name", action='store_true') + + clogging.add_argument(parser, loglevel_default='WARNING') + args = parser.parse_args() + clogging.setup(level=args.loglevel) + + config = configparser.ConfigParser() + + if args.rootconfig: + config.read(args.rootconfig) + os.environ[C.PLANTTRACER_CREDENTIALS] = args.rootconfig + + if args.readconfig: + paths.CREDENTIALS_FILE = paths.AWS_CREDENTIALS_FILE = args.readconfig + + if args.sendlink: + if not args.planttracer_endpoint: + raise RuntimeError("Please specify --planttracer_endpoint") + new_api_key = db.make_new_api_key(email=args.sendlink) + db.send_links(email=args.sendlink, planttracer_endpoint = args.planttracer_endpoint, + new_api_key=new_api_key) + sys.exit(0) + + ################################################################ + ## Startup stuff + + if args.createdb or args.dropdb: + cp = configparser.ConfigParser() + if args.rootconfig is None: + print("Please specify --rootconfig for --createdb or --dropdb",file=sys.stderr) + sys.exit(1) + + ath = dbfile.DBMySQLAuth.FromConfigFile(args.rootconfig, 'client') + with dbfile.DBMySQL( ath ) as droot: + if args.createdb: + createdb(droot=droot, createdb_name = args.createdb, + write_config_fname=args.writeconfig, schema=args.schema) + sys.exit(0) + + if args.dropdb: + # Delete the database and the users created for the database + dbreader_user = 'dbreader_' + args.dropdb + dbwriter_user = 'dbwriter_' + args.dropdb + c = droot.cursor() + for ipaddr in hostnames(): + c.execute(f'DROP USER IF EXISTS `{dbreader_user}`@`{ipaddr}`') + c.execute(f'DROP USER IF EXISTS `{dbwriter_user}`@`{ipaddr}`') + c.execute(f'DROP DATABASE IF EXISTS {args.dropdb}') + sys.exit(0) + + # These all use existing databases + cp = configparser.ConfigParser() + if args.create_client: + print(f"creating root with password '{args.create_client}'") + if 'client' not in cp: + cp.add_section('client') + cp['client']['user']='root' + cp['client']['password']=args.create_client + cp['client']['host'] = 'localhost' + cp['client']['database'] = 'sys' + + if args.readconfig: + cp.read(args.readconfig) + print("config read from",args.readconfig) + if cp['dbreader']['mysql_database'] != cp['dbwriter']['mysql_database']: + raise RuntimeError("dbreader and dbwriter do not address the same database") + + if args.writeconfig: + with open(args.writeconfig, 'w') as fp: + cp.write(fp) + print(args.writeconfig,"is written") + + if args.create_course: + print("creating course...") + if not args.admin_email: + print("Must provide --admin_email",file=sys.stderr) + if not args.admin_name: + print("Must provide --admin_name",file=sys.stderr) + if not args.admin_email or not args.admin_name: + sys.exit(1) + course_key = str(uuid.uuid4())[9:18] + dbmaint.create_course(course_key = course_key, + course_name = args.create_course, + admin_email = args.admin_email, + admin_name = args.admin_name, + max_enrollment = args.max_enrollment, + demo_email = args.demo_email + ) + print(f"course_key: {course_key}") + sys.exit(0) + + if args.upgradedb: + # the upgrade can be done with dbwriter, as long as dbwriter can update the schema. + # In our current versions, it can. + ath = dbfile.DBMySQLAuth.FromConfigFile(os.environ[C.PLANTTRACER_CREDENTIALS], 'dbwriter') + dbmaint.schema_upgrade(ath) + sys.exit(0) + + if args.add_admin: + print("adding admin to course...") + if not args.admin_email: + print("Must provide --admin_email",file=sys.stderr) + sys.exit(1) + user = db.lookup_user(email=args.admin_email) + if not user.get('id'): + print(f"User {args.admin_email} does not exist") + sys.exit(1) + if not args.course_key and not args.course_id and not args.course_name: + print("Must provide one of --course_key, --course_id, or --course_name",file=sys.stderr) + sys.exit(1) + if args.course_id: + course = db.lookup_course_by_id(course_id=args.course_id) + if course.get('id'): + dbmaint.add_admin_to_course(admin_email = args.admin_email, course_id = args.course_id) + sys.exit(0) + else: + print(f"Course with id {args.course_id} does not exist.",file=sys.stderr) + sys.exit(1) + elif args.course_key: + course = db.lookup_course_by_key(course_key=args.course_key) + if course.get('course_key'): + dbmaint.add_admin_to_course(admin_email = args.admin_email, course_key = course['course_key']) + sys.exit(0) + else: + print(f"Course with key {args.course_key} does not exist.",file=sys.stderr) + sys.exit(1) + elif args.course_name: + course = db.lookup_course_by_name(course_name = args.course_name) + if course.get('id'): + dbmaint.add_admin_to_course(admin_email=args.admin_email, course_id=course['id']) + sys.exit(0) + else: + print(f'Course with name {args.course_name} does not exist.',file=sys.stderr) + sys.exit(1) + + if args.remove_admin: + print("removing admin from course...") + if not args.admin_email: + print("Must provide --admin_email",file=sys.stderr) + sys.exit(1) + user = db.lookup_user(email=args.admin_email) + if not user.get('id'): + print(f"User {args.admin_email} does not exist") + sys.exit(1) + if not args.course_key and not args.course_id and not args.course_name: + print("Must provide one of --course_key, --course_id, or --course_name",file=sys.stderr) + sys.exit(1) + dbmaint.remove_admin_from_course( + admin_email = args.admin_email, + course_id = args.course_id, + course_key = args.course_key, + course_name = args.course_name + ) + sys.exit(0) + + ################################################################ + ## Cleanup + + if args.purge_test_data: + dbmaint.purge_test_data() + + if args.purge_all_movies: + dbmaint.purge_all_movies() + + if args.purge_movie: + db.purge_movie(movie_id=args.purge_movie) + + ################################################################ + ## Maintenance + + if args.report: + dbmaint.report() + sys.exit(0) + + if args.freshen: + dbmaint.freshen(False) + sys.exit(0) + + if args.clean: + dbmaint.freshen(True) + sys.exit(0) + + if args.dump: + dbmaint.dump(config,args.dump) + sys.exit() diff --git a/deploy/app/apikey.py b/deploy/app/apikey.py index d4aab7a0..02bec5ee 100644 --- a/deploy/app/apikey.py +++ b/deploy/app/apikey.py @@ -2,6 +2,7 @@ apikey.py Implements the user_dict and APIKEY functions - the high-level authentication system. +All done through get_user_dict() below, which is kind of gross. """ @@ -80,6 +81,14 @@ def cookie_name(): return C.API_KEY_COOKIE_BASE + "-" + get_dbreader().database +def add_cookie(response): + """Add the cookie if the apikey was in the get value""" + api_key = request.values.get('api_key', None) + if api_key: + response.set_cookie(cookie_name(), api_key, + max_age = C.API_KEY_COOKIE_MAX_AGE) + + def get_user_api_key(): """Gets the user APIkey from either the URL or the cookie or the form, but does not validate it. If we are running in an @@ -92,10 +101,9 @@ def get_user_api_key(): a string of the demo mode's API key if no user is logged in and demo mode is available. None if user is not logged in and no demo mode """ - # check the query string + # check the query string. api_key = request.values.get('api_key', None) # must be 'api_key', because may be in URL if api_key is not None: - logging.debug("api_key set in request.values=%s",api_key) return api_key # Return the api_key if it is in a cookie. diff --git a/deploy/app/auth.py b/deploy/app/auth.py index 06890c42..fb6ee772 100644 --- a/deploy/app/auth.py +++ b/deploy/app/auth.py @@ -15,7 +15,6 @@ from .constants import C from .paths import DEFAULT_CREDENTIALS_FILE -COOKIE_MAXAGE = 60*60*24*180 SMTP_ATTRIBS = ['SMTP_USERNAME', 'SMTP_PASSWORD', 'SMTP_PORT', 'SMTP_HOST'] ################################################################ diff --git a/deploy/app/bottle_api.py b/deploy/app/bottle_api.py index 08bae298..abff30a8 100644 --- a/deploy/app/bottle_api.py +++ b/deploy/app/bottle_api.py @@ -14,7 +14,7 @@ from collections import defaultdict from zipfile import ZipFile -from flask import Blueprint, request, make_response, redirect +from flask import Blueprint, request, make_response, redirect, current_app from validate_email_address import validate_email from . import db @@ -267,6 +267,9 @@ def api_upload_movie(): :param: key - where the file gets uploaded -from api_new_movie() :param: request.files['file'] - the file! """ + print("HELLO",file=sys.stderr) + current_app.logger.info("info") + current_app.logger.error("error") scheme = get('scheme') key = get('key') movie_data_sha256 = get('sha256') # claimed SHA256 diff --git a/deploy/app/bottle_app.py b/deploy/app/bottle_app.py index a93aff95..4916e5aa 100644 --- a/deploy/app/bottle_app.py +++ b/deploy/app/bottle_app.py @@ -10,6 +10,8 @@ import logging from flask import Flask, request, render_template, jsonify, make_response +from flask.logging import default_handler +from logging.config import dictConfig # Bottle creates a large number of no-member errors, so we just remove the warning # pylint: disable=no-member @@ -17,6 +19,7 @@ from . import db_object from . import dbmaint from . import clogging +from . import apikey from .bottle_api import api_bp from .constants import __version__,GET,GET_POST,C @@ -55,12 +58,32 @@ def lambda_startup(): ################################################################ ## API SUPPORT +dictConfig({ + 'version': 1, + 'formatters': {'default': { + 'format': '[%(asctime)s] [%(process)d] %(levelname)s %(filename)s:%(lineno)d %(message)s', + }}, + 'handlers': {'wsgi': { + 'class': 'logging.StreamHandler', + 'stream': 'ext://flask.logging.wsgi_errors_stream', + 'formatter': 'default' + }}, + 'root': { + 'level': 'DEBUG', + 'handlers': ['wsgi'] + } +}) + app = Flask(__name__) app.register_blueprint(api_bp, url_prefix='/api') +root = logging.getLogger() +fix_boto_log_level() +app.logger.info("Application logging is configured. __name__=%s",__name__) -# Note - Flask automatically serves /static -## Error handling +################################################################ +### Error Handling +################################################################ @app.errorhandler(AuthError) def handle_auth_error(ex): @@ -76,6 +99,9 @@ def handle_auth_error(ex): # HTML Pages served with template system ################################################################ +################ +## These mostly do forms or static content + @app.route('/', methods=GET) def func_root(): """/ - serve the home page""" @@ -93,19 +119,10 @@ def func_error(): def func_audit(): return render_template('audit.html', **page_dict("Audit", require_auth=True)) -@app.route('/list', methods=GET) -def func_list(): - return render_template('list.html', **page_dict('List Movies', require_auth=True)) - @app.route('/analyze', methods=GET) def func_analyze(): return render_template('analyze.html', **page_dict('Analyze Movie', require_auth=True)) -## debug page -@app.route('/debug', methods=GET) -def app_debug(): - return render_template('debug.html', routes=app.url_map) - ## ## Login page includes the api keys of all the demo users. ## @@ -134,7 +151,6 @@ def func_register(): hostname=request.host, register=True) - @app.route('/resend', methods=GET) def func_resend(): """/resend sends the register.html template which loads register.js with register variable set to False""" @@ -147,20 +163,39 @@ def func_resend(): def func_tos(): return render_template('tos.html', **page_dict('Terms of Service')) -@app.route('/upload', methods=GET) -def func_upload(): - """/upload - Upload a new file""" - logging.debug("/upload require_auth=True") - return render_template('upload.html', **page_dict('Upload a Movie', require_auth=True)) - @app.route('/users', methods=GET) def func_users(): """/users - provide a users list""" return render_template('users.html', **page_dict('List Users', require_auth=True)) +################ +# These are the two links that might have an ?apikey=; if we got that, set the cookie +@app.route('/list', methods=GET) +def func_list(): + response = make_response(render_template('list.html', + **page_dict('List Movies', + require_auth=True))) + # if api_key was in the query string, set the cookie + apikey.add_cookie(response) + return response + +@app.route('/upload', methods=GET) +def func_upload(): + """/upload - Upload a new file. Can also set cookie.""" + logging.debug("/upload require_auth=True") + response = make_response(render_template('upload.html', + **page_dict('Upload a Movie', + require_auth=True))) + apikey.add_cookie(response) + return response + ################################################################ ## debug/demo +@app.route('/debug', methods=GET) +def app_debug(): + return render_template('debug.html', routes=app.url_map) + @app.route('/demo_tracer1.html', methods=GET) def demo_tracer1(): return render_template('demo_tracer1.html', **page_dict('demo_tracer1',require_auth=False)) diff --git a/deploy/app/constants.py b/deploy/app/constants.py index 3fa34310..0e8717cc 100644 --- a/deploy/app/constants.py +++ b/deploy/app/constants.py @@ -27,7 +27,7 @@ class C: API_BASE='API_BASE' STATIC_BASE='STATIC_BASE' TRACKING_COMPLETED='TRACKING COMPLETED' # keep case; it's used as a flag - MAX_FILE_UPLOAD = 1024*1024*64 + MAX_FILE_UPLOAD = 1024*1024*256 MAX_FRAMES = 1e6 # max possible frames in a movie NOTIFY_UPDATE_INTERVAL = 5.0 TRACK_DELAY = 'TRACK_DELAY' @@ -44,6 +44,7 @@ class C: SCHEME_DB_MAX_OBJECT_LEN = 16_000_000 REDIRECT_FOUND = 302 API_KEY_COOKIE_BASE = 'api_key' + API_KEY_COOKIE_MAX_AGE = 60*60*24*180 SMTPCONFIG_ARN = 'SMTPCONFIG_ARN' class MIME: diff --git a/deploy/app/dbmaint.py b/deploy/app/dbmaint.py index ea17c058..69d1c740 100755 --- a/deploy/app/dbmaint.py +++ b/deploy/app/dbmaint.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -Database Management Tool for webapp +Database Management Support """ import sys @@ -18,14 +18,11 @@ from tabulate import tabulate from botocore.exceptions import ClientError,ParamValidationError -from . import paths from . import db from . import tracker from . import auth -from . import clogging from . import dbfile -from .constants import C -from .paths import TEMPLATE_DIR, SCHEMA_FILE, TEST_DATA_DIR, SCHEMA_TEMPLATE +from .paths import TEMPLATE_DIR, TEST_DATA_DIR, SCHEMA_TEMPLATE from .dbfile import MYSQL_HOST,MYSQL_USER,MYSQL_PASSWORD,MYSQL_DATABASE,DBMySQL assert os.path.exists(TEMPLATE_DIR) @@ -361,227 +358,3 @@ def dump(config,dumpdir): json.dump(movie, f, default=str) with open(os.path.join(dumpdir,f"movie_{movie_id}.mp4"),"wb") as f: f.write(movie_data) - -if __name__ == "__main__": - import argparse - parser = argparse.ArgumentParser(description="Database Maintenance Program. The database to act upon is specified in the ini file specified by the PLANTTRACER_CREDENTIALS environment variable, in the sections for [dbreader] and [dbwriter]", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - required = parser.add_argument_group('required arguments') - required.add_argument( - "--rootconfig", - help='Specify config file with MySQL database root credentials in [client] section. ' - 'Format is the same as the mysql --defaults-extra-file= argument') - parser.add_argument("--sendlink", help="Send link to the given email address, registering it if necessary.") - parser.add_argument('--planttracer_endpoint',help='https:// endpoint where planttracer app can be found') - parser.add_argument("--createdb", - help='Create a new database and a dbreader and dbwriter user. Database must not exist. ' - 'Requires that the variables MYSQL_DATABASE and MYSQL_HOST are set, and that MYSQL_PASSWORD and MYSQL_USER ' - 'are set with a MySQL username that can issue the "CREATE DATABASE" command.' - 'Outputs setenv for DBREADER and DBWRITER') - parser.add_argument("--upgradedb", help='Upgrade a database schema',action='store_true') - parser.add_argument("--dropdb", help='Drop an existing database.') - parser.add_argument("--readconfig", help="Specify the config.ini file to read") - parser.add_argument("--writeconfig", help="Specify the config.ini file to write.") - parser.add_argument('--purge_test_data', help='Remove the test data from the database', action='store_true') - parser.add_argument('--purge_all_movies', help='Remove all of the movies from the database', action='store_true') - parser.add_argument("--purge_movie",help="Remove the movie and all of its associated data from the database",type=int) - parser.add_argument("--create_client",help="Create a [client] section with a root username and the specified password") - parser.add_argument("--create_course",help="Create a course and register --admin_email --admin_name as the administrator") - parser.add_argument('--demo_email',help='If create_course is specified, also create a demo user with this email and upload demo movies ') - parser.add_argument("--admin_email",help="Specify the email address of the course administrator") - parser.add_argument("--admin_name",help="Specify the name of the course administrator") - parser.add_argument("--max_enrollment",help="Max enrollment for course",type=int,default=20) - parser.add_argument("--report",help="Print a report of the database",action='store_true') - parser.add_argument("--freshen",help="Non-destructive cleans up the movie metadata for all movies.",action='store_true') - parser.add_argument("--clean",help="Destructive cleans up the movie metadata for all movies.",action='store_true') - parser.add_argument("--schema", help="Specify schema file to use", default=SCHEMA_FILE) - parser.add_argument("--dump", help="Backup all objects as JSON files and movie files to new directory called DUMP. ") - parser.add_argument("--add_admin", help="Add --admin_email user as a course admin to the course specified by --course_id, --course_name, or --course_name", action='store_true') - parser.add_argument("--course_id", help="integer course id", type=int) - parser.add_argument("--course_key", help="integer course id") - parser.add_argument("--course_name", help="integer course id") - parser.add_argument("--remove_admin", help="Remove the --admin_email user as a course admin from the course specified by --course_id, --course_name, or --course_name", action='store_true') - - clogging.add_argument(parser, loglevel_default='WARNING') - args = parser.parse_args() - clogging.setup(level=args.loglevel) - - config = configparser.ConfigParser() - - if args.rootconfig: - config.read(args.rootconfig) - os.environ[C.PLANTTRACER_CREDENTIALS] = args.rootconfig - - if args.readconfig: - paths.CREDENTIALS_FILE = paths.AWS_CREDENTIALS_FILE = args.readconfig - - if args.sendlink: - if not args.planttracer_endpoint: - raise RuntimeError("Please specify --planttracer_endpoint") - new_api_key = db.make_new_api_key(email=args.sendlink) - db.send_links(email=args.sendlink, planttracer_endpoint = args.planttracer_endpoint, - new_api_key=new_api_key) - sys.exit(0) - - ################################################################ - ## Startup stuff - - if args.createdb or args.dropdb: - cp = configparser.ConfigParser() - if args.rootconfig is None: - print("Please specify --rootconfig for --createdb or --dropdb",file=sys.stderr) - sys.exit(1) - - ath = dbfile.DBMySQLAuth.FromConfigFile(args.rootconfig, 'client') - with dbfile.DBMySQL( ath ) as droot: - if args.createdb: - createdb(droot=droot, createdb_name = args.createdb, - write_config_fname=args.writeconfig, schema=args.schema) - sys.exit(0) - - if args.dropdb: - # Delete the database and the users created for the database - dbreader_user = 'dbreader_' + args.dropdb - dbwriter_user = 'dbwriter_' + args.dropdb - c = droot.cursor() - for ipaddr in hostnames(): - c.execute(f'DROP USER IF EXISTS `{dbreader_user}`@`{ipaddr}`') - c.execute(f'DROP USER IF EXISTS `{dbwriter_user}`@`{ipaddr}`') - c.execute(f'DROP DATABASE IF EXISTS {args.dropdb}') - sys.exit(0) - - # These all use existing databases - cp = configparser.ConfigParser() - if args.create_client: - print(f"creating root with password '{args.create_client}'") - if 'client' not in cp: - cp.add_section('client') - cp['client']['user']='root' - cp['client']['password']=args.create_client - cp['client']['host'] = 'localhost' - cp['client']['database'] = 'sys' - - if args.readconfig: - cp.read(args.readconfig) - print("config read from",args.readconfig) - if cp['dbreader']['mysql_database'] != cp['dbwriter']['mysql_database']: - raise RuntimeError("dbreader and dbwriter do not address the same database") - - if args.writeconfig: - with open(args.writeconfig, 'w') as fp: - cp.write(fp) - print(args.writeconfig,"is written") - - if args.create_course: - print("creating course...") - if not args.admin_email: - print("Must provide --admin_email",file=sys.stderr) - if not args.admin_name: - print("Must provide --admin_name",file=sys.stderr) - if not args.admin_email or not args.admin_name: - sys.exit(1) - course_key = str(uuid.uuid4())[9:18] - create_course(course_key = course_key, - course_name = args.create_course, - admin_email = args.admin_email, - admin_name = args.admin_name, - max_enrollment = args.max_enrollment, - demo_email = args.demo_email - ) - print(f"course_key: {course_key}") - sys.exit(0) - - if args.upgradedb: - # the upgrade can be done with dbwriter, as long as dbwriter can update the schema. - # In our current versions, it can. - ath = dbfile.DBMySQLAuth.FromConfigFile(os.environ[C.PLANTTRACER_CREDENTIALS], 'dbwriter') - schema_upgrade(ath) - sys.exit(0) - - if args.add_admin: - print("adding admin to course...") - if not args.admin_email: - print("Must provide --admin_email",file=sys.stderr) - sys.exit(1) - user = db.lookup_user(email=args.admin_email) - if not user.get('id'): - print(f"User {args.admin_email} does not exist") - sys.exit(1) - if not args.course_key and not args.course_id and not args.course_name: - print("Must provide one of --course_key, --course_id, or --course_name",file=sys.stderr) - sys.exit(1) - if args.course_id: - course = db.lookup_course_by_id(course_id=args.course_id) - if course.get('id'): - add_admin_to_course(admin_email = args.admin_email, course_id = args.course_id) - sys.exit(0) - else: - print(f"Course with id {args.course_id} does not exist.",file=sys.stderr) - sys.exit(1) - elif args.course_key: - course = db.lookup_course_by_key(course_key=args.course_key) - if course.get('course_key'): - add_admin_to_course(admin_email = args.admin_email, course_key = course['course_key']) - sys.exit(0) - else: - print(f"Course with key {args.course_key} does not exist.",file=sys.stderr) - sys.exit(1) - elif args.course_name: - course = db.lookup_course_by_name(course_name = args.course_name) - if course.get('id'): - add_admin_to_course(admin_email=args.admin_email, course_id=course['id']) - sys.exit(0) - else: - print(f'Course with name {args.course_name} does not exist.',file=sys.stderr) - sys.exit(1) - - if args.remove_admin: - print("removing admin from course...") - if not args.admin_email: - print("Must provide --admin_email",file=sys.stderr) - sys.exit(1) - user = db.lookup_user(email=args.admin_email) - if not user.get('id'): - print(f"User {args.admin_email} does not exist") - sys.exit(1) - if not args.course_key and not args.course_id and not args.course_name: - print("Must provide one of --course_key, --course_id, or --course_name",file=sys.stderr) - sys.exit(1) - remove_admin_from_course( - admin_email = args.admin_email, - course_id = args.course_id, - course_key = args.course_key, - course_name = args.course_name - ) - sys.exit(0) - - ################################################################ - ## Cleanup - - if args.purge_test_data: - purge_test_data() - - if args.purge_all_movies: - purge_all_movies() - - if args.purge_movie: - db.purge_movie(movie_id=args.purge_movie) - - ################################################################ - ## Maintenance - - if args.report: - report() - sys.exit(0) - - if args.freshen: - freshen(False) - sys.exit(0) - - if args.clean: - freshen(True) - sys.exit(0) - - if args.dump: - dump(config,args.dump) - sys.exit() diff --git a/deploy/app/paths.py b/deploy/app/paths.py index 835e41a9..24ffcfee 100644 --- a/deploy/app/paths.py +++ b/deploy/app/paths.py @@ -40,7 +40,7 @@ def ffmpeg_path(): if C.FFMPEG_PATH in os.environ: pth = os.environ[C.FFMPEG_PATH] - if os.path.exists(path): + if os.path.exists(pth): return pth pth = shutil.which('ffmpeg') if pth: diff --git a/etc/planttracer-template.service b/etc/planttracer-template.service index b87486e7..185a740d 100644 --- a/etc/planttracer-template.service +++ b/etc/planttracer-template.service @@ -11,9 +11,13 @@ Environment="PATH={base}/venv/bin" Environment="PLANTTRACER_CREDENTIALS={base}/deploy/etc/credentials-{name}.ini" Environment="DEMO_MODE={demo}" ExecStart={base}/venv/bin/gunicorn -w 2 -b 127.0.0.1:{port} \ - --access-logfile /home/ec2-user/logs/planttracer-{name}.log \ - --error-logfile /home/ec2-user/logs/planttracer-{name}-error.log\ + --access-logfile /home/ec2-user/logs/planttracer-{name}-access.log \ + --error-logfile /home/ec2-user/logs/planttracer-{name}-error.log \ --reload deploy.app.bottle_app:app +Restart=always +RestartSec=60 +StandardOutput=append:/home/ec2-user/logs/planttracer-{name}-error.log +StandardError=append:/home/ec2-user/logs/planttracer-{name}-error.log [Install] WantedBy=multi-user.target From 0f67c09ed41960814db151b19be86644bc947c6b Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Sat, 18 Jan 2025 22:12:39 +0000 Subject: [PATCH 18/22] pylint is clean --- deploy/app/bottle_app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/app/bottle_app.py b/deploy/app/bottle_app.py index 4916e5aa..ec1d8dfd 100644 --- a/deploy/app/bottle_app.py +++ b/deploy/app/bottle_app.py @@ -8,10 +8,9 @@ import sys import os import logging +from logging.config import dictConfig from flask import Flask, request, render_template, jsonify, make_response -from flask.logging import default_handler -from logging.config import dictConfig # Bottle creates a large number of no-member errors, so we just remove the warning # pylint: disable=no-member From a5c13ef3e0c57eaa9d1b29fa82cf6225d3552f94 Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Sat, 18 Jan 2025 23:46:22 +0000 Subject: [PATCH 19/22] upload works, tracking does not --- dbutil.py | 3 ++- deploy/app/bottle_api.py | 7 +++---- deploy/app/bottle_app.py | 21 ++++++++++++--------- deploy/app/db_object.py | 2 +- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/dbutil.py b/dbutil.py index 10bacee3..b9e9ece9 100644 --- a/dbutil.py +++ b/dbutil.py @@ -6,6 +6,7 @@ from deploy.app import dbmaint from deploy.app import db from deploy.app import dbfile +from deploy.app import paths if __name__ == "__main__": @@ -40,7 +41,7 @@ parser.add_argument("--report",help="Print a report of the database",action='store_true') parser.add_argument("--freshen",help="Non-destructive cleans up the movie metadata for all movies.",action='store_true') parser.add_argument("--clean",help="Destructive cleans up the movie metadata for all movies.",action='store_true') - parser.add_argument("--schema", help="Specify schema file to use", default=dbmaint.SCHEMA_FILE) + parser.add_argument("--schema", help="Specify schema file to use", default=paths.SCHEMA_FILE) parser.add_argument("--dump", help="Backup all objects as JSON files and movie files to new directory called DUMP. ") parser.add_argument("--add_admin", help="Add --admin_email user as a course admin to the course specified by --course_id, --course_name, or --course_name", action='store_true') parser.add_argument("--course_id", help="integer course id", type=int) diff --git a/deploy/app/bottle_api.py b/deploy/app/bottle_api.py index abff30a8..f4c8e0c7 100644 --- a/deploy/app/bottle_api.py +++ b/deploy/app/bottle_api.py @@ -267,9 +267,8 @@ def api_upload_movie(): :param: key - where the file gets uploaded -from api_new_movie() :param: request.files['file'] - the file! """ - print("HELLO",file=sys.stderr) - current_app.logger.info("info") - current_app.logger.error("error") + logging.info("info") + logging.error("error") scheme = get('scheme') key = get('key') movie_data_sha256 = get('sha256') # claimed SHA256 @@ -597,7 +596,7 @@ def api_ver(): """Report the python version. Allows us to validate we are using Python3. Run the dictionary below through the VERSION_TEAMPLTE with jinja2. """ - logging.debug("api_ver") + current_app.logger.error("api_ver") return {'__version__': __version__, 'sys_version': sys.version} ################################################################ diff --git a/deploy/app/bottle_app.py b/deploy/app/bottle_app.py index ec1d8dfd..55cff260 100644 --- a/deploy/app/bottle_app.py +++ b/deploy/app/bottle_app.py @@ -11,6 +11,7 @@ from logging.config import dictConfig from flask import Flask, request, render_template, jsonify, make_response +from flask.logging import default_handler # Bottle creates a large number of no-member errors, so we just remove the warning # pylint: disable=no-member @@ -45,14 +46,18 @@ def lambda_startup(): clogging.setup(level=os.environ.get('PLANTTRACER_LOG_LEVEL',logging.INFO)) fix_boto_log_level() + logging.info("p1") if os.environ.get(C.PLANTTRACER_S3_BUCKET,None): db_object.S3_BUCKET = os.environ[C.PLANTTRACER_S3_BUCKET] + logging.info("p2a %s",db_object.S3_BUCKET) else: config = auth.config() try: db_object.S3_BUCKET = config['s3']['s3_bucket'] + logging.info("p2b %s",db_object.S3_BUCKET) except KeyError as e: logging.info("s3_bucket not defined in config file. using db object store instead. %s",e) + logging.info("p3 %s",db_object.S3_BUCKET) ################################################################ ## API SUPPORT @@ -62,22 +67,19 @@ def lambda_startup(): 'formatters': {'default': { 'format': '[%(asctime)s] [%(process)d] %(levelname)s %(filename)s:%(lineno)d %(message)s', }}, - 'handlers': {'wsgi': { - 'class': 'logging.StreamHandler', - 'stream': 'ext://flask.logging.wsgi_errors_stream', - 'formatter': 'default' - }}, 'root': { 'level': 'DEBUG', - 'handlers': ['wsgi'] } }) +fix_boto_log_level() +lambda_startup() app = Flask(__name__) app.register_blueprint(api_bp, url_prefix='/api') -root = logging.getLogger() -fix_boto_log_level() -app.logger.info("Application logging is configured. __name__=%s",__name__) +app.logger.info("new Flask(__name__=%s)",__name__) +app.logger.info("PLANTTRACER_CREDENTIALS=%s",os.environ.get(C.PLANTTRACER_CREDENTIALS,None)) +app.logger.info("db_object.S3_BUCKET=%s",db_object.S3_BUCKET) +logging.info("regular logging works too") ################################################################ @@ -212,6 +214,7 @@ def func_ver(): """Demo for reporting python version. Allows us to validate we are using Python3. Run the dictionary below through the VERSION_TEAMPLTE with jinja2. """ + logging.info("/ver") response = make_response(render_template('version.txt', __version__=__version__, sys_version= sys.version)) diff --git a/deploy/app/db_object.py b/deploy/app/db_object.py index 45323c5f..82a78e8b 100644 --- a/deploy/app/db_object.py +++ b/deploy/app/db_object.py @@ -134,7 +134,7 @@ def read_signed_url(*,urn,sig): logging.error("URL signature does not match. urn=%s sig=%s computed_sig=%s",urn,sig,computed_sig) raise AuthError("signature does not verify") -def make_presigned_post(*, urn, maxsize=10_000_000, mime_type='video/mp4',expires=3600, sha256=None): +def make_presigned_post(*, urn, maxsize=C.MAX_FILE_UPLOAD, mime_type='video/mp4',expires=3600, sha256=None): """Returns a dictionary with 'url' and 'fields'""" o = urllib.parse.urlparse(urn) if o.scheme==C.SCHEME_S3: From 0f8f8c1e38b2f8f8d2aab3c629d036dc31e65d1d Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Sat, 25 Jan 2025 13:26:44 -0500 Subject: [PATCH 20/22] update makefile for running dbutil.py and not dbmaint.py module --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2cb18e13..c0f21010 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ PYTHON=venv/bin/python PIP_INSTALL=venv/bin/pip install --no-warn-script-location ROOT_ETC=etc DEPLOY_ETC=deploy/etc -DBMAINT=-m deploy.app.dbmaint +DBMAINT=dbutil.py # Note: PLANTTRACER_CREDENTIALS must be set @@ -207,8 +207,10 @@ jscoverage: PLANTTRACER_LOCALDB_NAME ?= actions_test create_localdb: - @echo Creating local database, exercise the upgrade code and write credentials to $(PLANTTRACER_CREDENTIALS) using $(ROOT_ETC)/github_actions_mysql_rootconfig.ini + @echo Creating local database, exercise the upgrade code and write credentials + @echo to $(PLANTTRACER_CREDENTIALS) using $(ROOT_ETC)/github_actions_mysql_rootconfig.ini @echo $(PLANTTRACER_CREDENTIALS) will be used automatically by other tests + pwd mkdir -p $(ROOT_ETC) ls -l $(ROOT_ETC) $(PYTHON) $(DBMAINT) --create_client=$$MYSQL_ROOT_PASSWORD --writeconfig $(ROOT_ETC)/github_actions_mysql_rootconfig.ini From 82e87ea65e064eda0740d7dc2b6f2c5d359b08ef Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Sat, 25 Jan 2025 13:34:17 -0500 Subject: [PATCH 21/22] added import --- dbutil.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbutil.py b/dbutil.py index b9e9ece9..7644b31d 100644 --- a/dbutil.py +++ b/dbutil.py @@ -7,6 +7,7 @@ from deploy.app import db from deploy.app import dbfile from deploy.app import paths +from deploy.app.constants import C if __name__ == "__main__": From b5db967aaf36c8444f2aaab62c9b74d9514f4b30 Mon Sep 17 00:00:00 2001 From: Simson Garfinkel Date: Sat, 25 Jan 2025 13:53:46 -0500 Subject: [PATCH 22/22] cleaned up pylint errors and dbmaint/dbutil mixups --- Makefile | 1 + dbutil.py | 9 +++++++-- deploy/app/bottle_app.py | 1 - docs/mv1.rst | 10 +++++----- etc/planttracer-template.service | 2 +- standalone.py | 12 +++++++----- 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index c0f21010..b4730143 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,7 @@ check: PYLINT_OPTS:=--output-format=parseable --rcfile .pylintrc --fail-under=$(PYLINT_THRESHOLD) --verbose pylint: $(REQ) $(PYTHON) -m pylint $(PYLINT_OPTS) deploy + $(PYTHON) -m pylint $(PYLINT_OPTS) *.py pylint-tests: $(REQ) $(PYTHON) -m pylint $(PYLINT_OPTS) --init-hook="import sys;sys.path.append('tests');import conftest" tests diff --git a/dbutil.py b/dbutil.py index 7644b31d..faa6cbf9 100644 --- a/dbutil.py +++ b/dbutil.py @@ -1,6 +1,11 @@ +""" +dbutil.py - CLI for dbmaint module. +""" + import sys import os import configparser +import uuid from deploy.app import clogging from deploy.app import dbmaint @@ -83,7 +88,7 @@ ath = dbfile.DBMySQLAuth.FromConfigFile(args.rootconfig, 'client') with dbfile.DBMySQL( ath ) as droot: if args.createdb: - createdb(droot=droot, createdb_name = args.createdb, + dbmaint.createdb(droot=droot, createdb_name = args.createdb, write_config_fname=args.writeconfig, schema=args.schema) sys.exit(0) @@ -92,7 +97,7 @@ dbreader_user = 'dbreader_' + args.dropdb dbwriter_user = 'dbwriter_' + args.dropdb c = droot.cursor() - for ipaddr in hostnames(): + for ipaddr in dbmaint.hostnames(): c.execute(f'DROP USER IF EXISTS `{dbreader_user}`@`{ipaddr}`') c.execute(f'DROP USER IF EXISTS `{dbwriter_user}`@`{ipaddr}`') c.execute(f'DROP DATABASE IF EXISTS {args.dropdb}') diff --git a/deploy/app/bottle_app.py b/deploy/app/bottle_app.py index 55cff260..e0aaa3b3 100644 --- a/deploy/app/bottle_app.py +++ b/deploy/app/bottle_app.py @@ -11,7 +11,6 @@ from logging.config import dictConfig from flask import Flask, request, render_template, jsonify, make_response -from flask.logging import default_handler # Bottle creates a large number of no-member errors, so we just remove the warning # pylint: disable=no-member diff --git a/docs/mv1.rst b/docs/mv1.rst index 023431a2..7a91235a 100644 --- a/docs/mv1.rst +++ b/docs/mv1.rst @@ -1,7 +1,7 @@ Configuration on mv1: -Host | gunicorn port | app dir | app name -mv1.planttracer.com | 8000 | /home/ec2-user/webapp/deploy/ | app.app -app.planttracer.com | 8010 | /home/ec2-user/webapp/deploy/ | app.app -demo1.planttracer.com | 8020 | /home/ec2-user/webapp/deploy/ | app.app -dev-slg.planttracer.com | 8030| /home/ec2-user/slg-dev/deploy/ | app.app +Host | gunicorn port | app dir | app name +mv1.planttracer.com | 8000 | /home/ec2-user/webapp/deploy/ | app.app +app.planttracer.com | 8010 | /home/ec2-user/webapp/deploy/ | app.app +demo1.planttracer.com | 8020 | /home/ec2-user/webapp/deploy/ | app.app +dev-slg.planttracer.com | 8030 | /home/ec2-user/slg-dev/deploy/ | app.app diff --git a/etc/planttracer-template.service b/etc/planttracer-template.service index 185a740d..8b606157 100644 --- a/etc/planttracer-template.service +++ b/etc/planttracer-template.service @@ -16,7 +16,7 @@ ExecStart={base}/venv/bin/gunicorn -w 2 -b 127.0.0.1:{port} \ --reload deploy.app.bottle_app:app Restart=always -RestartSec=60 +RestartSec=10 StandardOutput=append:/home/ec2-user/logs/planttracer-{name}-error.log StandardError=append:/home/ec2-user/logs/planttracer-{name}-error.log [Install] diff --git a/standalone.py b/standalone.py index 52b3c3e3..63fbc512 100755 --- a/standalone.py +++ b/standalone.py @@ -1,4 +1,6 @@ -#!/usr/bin/env python3.11 +""" +CLI for running standalone webserver. +""" ################################################################ # Bottle App @@ -6,11 +8,12 @@ import sys import os -import uvicorn import argparse +import logging -import deploy.app.clogging as clogging +from deploy.app import clogging from deploy.app.constants import C +from deploy.app import db_object if __name__ == "__main__": parser = argparse.ArgumentParser(description="Run Bottle App with Bottle's built-in server unless a command is given", @@ -27,7 +30,7 @@ sys.exit(1) if args.info: - for name in logging.root.manager.loggerDict: + for name in logging.root.manager.loggerDict: # pylint: disable=no-member print("Logger: ",name) sys.exit(0) @@ -45,5 +48,4 @@ cmd = f'gunicorn --bind 127.0.0.1:{args.port} --workers 2 --reload --log-level DEBUG deploy.app.bottle_app:app ' print(cmd) - exit(0) os.system(cmd)