Merge pull request #17 from fecgov/feature/40-test-cloud-gov

Deploy to cloud.gov
fecgov · Feb 7, 2022 · 735c198 · 735c198
2 parents fee05bf + 423949e
commit 735c198
Show file tree

Hide file tree

Showing 25 changed files with 436 additions and 272 deletions.
diff --git a/.cfignore b/.cfignore
@@ -0,0 +1,5 @@
+# Database creation script
+db
+
+# git secrets config
+install-git-secrets-hook.sh
diff --git a/.circleci/README.md b/.circleci/README.md
@@ -1,19 +1,29 @@
 # CircleCI Configuration
 ## Environment Variables
-When configuring CircleCI, you will need to set environment varaialbes the database
+When configuring CircleCI, you will need to set environment variables the database
 configuration as follows:
 ```
-FECFILE_DB_HOST=localhost 
-FECFILE_DB_USERNAME=postgres 
-FECFILE_DB_PASSWORD=postgres 
-FECFILE_DB_NAME=postgres 
+DATABASE_URL = "postgres://postgres:postgres@0.0.0.0/postgres"
+FECFILE_TEST_DB_NAME = "postgres"
 FECFILE_FEC_WEBSITE_API_KEY=
 ```
 Notes:
 * There is no default FECFILE_FEC_WEBSITE_API_KEY, you must obtain and set this yourself
-* The FECFILE_DB_HOST value here is different than what you need for your local docker-compose configuration.
 
-# Using CircleCI local CLI 
+CircleCI will attempt to deploy commits made to specific branches:
+* branch __develop__ -> cloud.gov dev space
+* branch __release__* (any branch starting with release) -> cloud.gov staging space
+* branch __prod__ -> cloud.gov prod space
+
+Authentication must be configured in a set of evironment variables:
+* $FEC_CF_USERNAME_DEV
+* $FEC_CF_PASSWORD_DEV
+* $FEC_CF_USERNAME_STAGE
+* $FEC_CF_PASSWORD_STAGE
+* $FEC_CF_USERNAME_PROD
+* $FEC_CF_PASSWORD_PROD
+
+# Using CircleCI local CLI
 
 ## Install circleci local
 Install on Linux or Mac with:
@@ -30,23 +40,21 @@ circleci config validate
 ```
 
 ## Run the CircleCI Job locally
-You can run a CircleCI job locally and avoid the change/commit/wait loop you need to 
-do if you want to actually run the changes on Circle. 
+You can run a CircleCI job locally and avoid the change/commit/wait loop you need to
+do if you want to actually run the changes on Circle.
 This can save a lot of time when trying to debug an issue in CI.
 ```
 circleci local execute --job JOB_NAME
 ```
 
 ## Necessary Environment Variables
-The Django backend expects to find the database login info in the environment.  
+The Django backend expects to find the database login info in the environment.
 To run in the local CircleCI for the django unit tests (for example), use the following:
 
 ```
-circleci local execute -e FECFILE_DB_HOST=localhost \
-         -e FECFILE_DB_USERNAME=postgres \
-         -e FECFILE_DB_PASSWORD=postgres \
-         -e FECFILE_DB_NAME=postgres \
-         -e FECFILE_FEC_WEBSITE_API_KEY=${FECFILE_FEC_WEBSITE_API_KEY}\
+circleci local execute -e DATABASE_URL=${DATABASE_URL} \
+         -e FECFILE_FEC_WEBSITE_API_KEY=${FECFILE_FEC_WEBSITE_API_KEY} \
+         -e FECFILE_TEST_DB_NAME=${FECFILE_TEST_DB_NAME} \
          --job unit-test
 ```
 

diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -13,14 +13,15 @@ jobs:
 
   dependency-check:
     docker:
-      - image: cimg/python:3.8
+      - image: cimg/python:3.7
 
     steps:
       - checkout
 
       - python/install-packages:
           pkg-manager: pip
-          pip-dependency-file: django-backend/requirements-test.txt
+          pip-dependency-file: requirements-test.txt
+
 
       - run:
           name: Run depency check
@@ -34,7 +35,7 @@ jobs:
               awk -v "today=${today}" '{ if ($2 > today || $2 == "") print "-i", $1}' | # print any line after today
               xargs echo  # put all the output from previous command on one line
             )
-            export command="safety check -r django-backend/requirements.txt --full-report $ignores"
+            export command="safety check -r requirements.txt --full-report $ignores"
 
             echo "----------------------------------------------------"
             echo "If you need to modify the ignore list for the safety"
@@ -68,29 +69,28 @@ jobs:
       - run:
           name: Create unified requirements so CircleCI can cache them
           command: |
-            cd ~/project/django-backend
+            cd ~/project/
             ls -l
             cat requirements.txt > requirements-all.txt
             echo >> requirements-all.txt  # blank in case new newline at end of requirements.txt
             cat requirements-test.txt >> requirements-all.txt
 
       - python/install-packages:
           pkg-manager: pip
-          app-dir: ~/project/django-backend/
+          app-dir: ~/project/
           pip-dependency-file: requirements-all.txt
 
       - run:
           name: Wait for the database to be active
-          command: pip install psycopg2 psycopg2-binary retry &&  python wait_for_db.py
+          command: python wait_for_db.py
           working_directory: ~/project/django-backend/
 
       - run:
           name: Load test database fixure
           command: |
             sudo apt-get update &&
             sudo apt-get install postgresql-client-12 &&
-            export PGPASSWORD=${FECFILE_DB_PASSWORD} &&
-            psql -h ${FECFILE_DB_HOST}  ${FECFILE_DB_NAME} ${FECFILE_DB_USERNAME} < fec_clean_dev_db_backup-20211227.sql
+            psql ${DATABASE_URL} < fec_clean_dev_db_backup-20211227.sql
           working_directory: ~/project/db
 
       - run:
@@ -127,7 +127,6 @@ jobs:
           command: coverage xml -o ~/project/coverage-reports/coverage.xml
           working_directory: ~/project/django-backend
 
-
       # Sonar cloud setup and scanning
       - run:
           name: Create sonar-scanner cache directory if it doesn't exist
@@ -162,12 +161,39 @@ jobs:
           key: v1-sonarcloud-scanner-4.6.2.2472
           paths: /tmp/cache/scanner
 
+  deploy:
+
+    docker:
+      - image: cimg/python:3.7
+
+    steps:
+      - checkout
+
+      - python/install-packages:
+          pkg-manager: pip
+          pip-dependency-file: requirements.txt
+
+      - run:
+          name: Installs for deploy
+          command: |
+            mkdir -p $HOME/bin
+            export PATH=$HOME/bin:$PATH
+            curl -L "https://cli.run.pivotal.io/stable?release=linux64-binary&version=7.1.0" | tar xzv -C $HOME/bin
+
+      - deploy:
+          name: Deploy API
+          command: |
+            export PATH=$HOME/bin:$PATH
+            invoke deploy --branch $CIRCLE_BRANCH --login
+
 # Invoke jobs via workflows
 # See: https://circleci.com/docs/2.0/configuration-reference/#workflows
 workflows:
   test: # This is the name of the workflow, feel free to change it to better match your workflow.
     jobs:
       - test
       - dependency-check
-
-
+      - deploy:
+          requires:
+              - test
+              - dependency-check
diff --git a/.gitignore b/.gitignore
@@ -67,3 +67,6 @@ testem.log
 Thumbs.db
 .vscode/settings.json
 front-end/.vscode/launch.json
+
+# cloud foundry
+.cfmeta
diff --git a/django-backend/Dockerfile → Dockerfile b/django-backend/Dockerfile → Dockerfile
@@ -3,13 +3,13 @@ ENV PYTHONUNBUFFERED=1
 
 RUN mkdir /opt/nxg_fec
 WORKDIR /opt/nxg_fec
-ADD requirements.txt /opt/nxg_fec/
-RUN pip3 install -r requirements.txt
+ADD requirements.txt /opt
+RUN pip3 install -r /opt/requirements.txt
 
 RUN mv /etc/localtime /etc/localtime.backup && ln -s /usr/share/zoneinfo/EST5EDT /etc/localtime
 
 RUN useradd nxgu --no-create-home --home /opt/nxg_fec && chown -R nxgu:nxgu /opt/nxg_fec
 user nxgu
 
 EXPOSE 8080
-ENTRYPOINT ["sh", "-c", "pip3 install -r requirements.txt && python wait_for_db.py && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 10 -t 200 --reload"]
+ENTRYPOINT ["sh", "-c", "cp /opt/requirements.txt . && pip3 install -r requirements.txt && python wait_for_db.py && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 10 -t 200 --reload"]
diff --git a/Procfile b/Procfile
@@ -0,0 +1 @@
+web: bin/run.sh
diff --git a/README.md b/README.md
@@ -30,11 +30,9 @@ When running docker-compose you will need to be in the root directory of the pro
 You should set the following environment variables in the shell where you are running 'docker-compose up -d'.
 Proper values for the development variables are shown here as an example
 ```
-export FECFILE_DB_HOST=db
-export FECFILE_DB_USERNAME=postgres
-export FECFILE_DB_PASSWORD=postgres
-export FECFILE_DB_NAME=postgres
-export FECFILE_URL=localhost
+export DATABASE_URL = "postgres://postgres:postgres@0.0.0.0/postgres"
+export FECFILE_TEST_DB_NAME = "postgres"
+export DJANGO_SECRET_KEY = "If_using_test_db_use_secret_key_in_cloud.gov"
 ```
 ### Shut down the containers
 `docker-compose down`

diff --git a/bin/run.sh b/bin/run.sh
@@ -0,0 +1,8 @@
+cd django-backend
+
+# Run migrations
+./manage.py makemigrations
+./manage.py migrate --noinput
+
+# Run application
+python wait_for_db.py && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 9 -t 200 --reload
diff --git a/django-backend/fecfiler/core/transactions_chk_csv_duplicates.py b/django-backend/fecfiler/core/transactions_chk_csv_duplicates.py
@@ -1,4 +1,3 @@
-import hashlib
 import os
 import psycopg2
 import pandas as pd
@@ -7,6 +6,7 @@
 from pandas.util import hash_pandas_object
 import numpy
 from psycopg2.extensions import register_adapter, AsIs
+from django.conf import settings
 
 
 def addapt_numpy_float64(numpy_float64):
@@ -20,12 +20,6 @@ def addapt_numpy_int64(numpy_int64):
 register_adapter(numpy.float64, addapt_numpy_float64)
 register_adapter(numpy.int64, addapt_numpy_int64)
 
-
-PG_HOST = os.getenv('FECFILE_DB_HOST', 'localhost')
-PG_PORT = os.getenv('FECFILE_DB_PORT', 5432)
-PG_DATABASE = os.getenv('FECFILE_DB_NAME', 'postgres')
-PG_USER = os.getenv('FECFILE_DB_USERNAME', 'postgres')
-PG_PASSWORD = os.getenv('FECFILE_DB_PASSWORD', 'postgres')
 SQS_QUEUE_NAME = os.getenv('SQS_QUEUE_NAME')
 
 
@@ -50,13 +44,7 @@ def check_for_file_hash_in_db(cmteid, filename, hash, fecfilename):
         """ insert a transactions_file_details """
         selectsql = """SELECT cmte_id, md5, file_name, create_date FROM public.transactions_file_details WHERE cmte_id = %s AND file_name = %s AND md5 = %s AND fec_file_name = %s;"""
 
-        conn = psycopg2.connect(
-            user=PG_USER,
-            password=PG_PASSWORD,
-            host=PG_HOST,
-            port=PG_PORT,
-            database=PG_DATABASE
-        )
+        conn = psycopg2.connect(settings.DATABASE_URL)
         cur = conn.cursor()
         cur.execute(selectsql, (cmteid, filename, hash, fecfilename))
         dbhash = cur.fetchone()
@@ -82,13 +70,7 @@ def load_file_hash_to_db(cmteid, filename, hash, fecfilename):
         insertsql = """INSERT INTO transactions_file_details(cmte_id, file_name, md5, fec_file_name)
                 VALUES(%s, %s, %s, %s);"""
 
-        conn = psycopg2.connect(
-            user=PG_USER,
-            password=PG_PASSWORD,
-            host=PG_HOST,
-            port=PG_PORT,
-            database=PG_DATABASE
-        )
+        conn = psycopg2.connect(settings.DATABASE_URL)
         cur = conn.cursor()
         cur.execute(insertsql, (cmteid, filename, hash, fecfilename))
         conn.commit()
@@ -140,17 +122,17 @@ def file_verify_upload(request):
             bucket = AWS_STORAGE_IMPORT_CONTACT_BUCKET_NAME
             file_name = request.data.get("fileName")
             csv_obj = client.get_object(Bucket=bucket, Key=file_name)
-            hashlib.sha1(pd.util.hash_pandas_object(df).values).hexdigest() 
+            hashlib.sha1(pd.util.hash_pandas_object(df).values).hexdigest()
 
             #filepath = dirname(dirname(os.getcwd()))+"/csv/"
             filename = "Disbursements_1q2020.csv"
             hash_value = generate_md5_hash(filepath+filename)
             fileexists = check_for_file_hash_in_db('C00000018', filename, hash_value)
-            if fileexists is None: 
+            if fileexists is None:
                 load_file_hash_to_db('C00000018', filename, hash_value)
                 print('File loaded successfully!!!')
             else:
-                print('File exists in DB')    
+                print('File exists in DB')
 
             return JsonResponse(contacts, status=status.HTTP_201_CREATED, safe=False)
 

diff --git a/django-backend/fecfiler/core/transactions_validate_csv.py b/django-backend/fecfiler/core/transactions_validate_csv.py
@@ -1,35 +1,23 @@
-import hashlib
 import os
 import os.path
 from os import path
 import psycopg2
 import pandas as pd
 from pandas_schema import Column, Schema
-from pandas_schema.validation import MatchesPatternValidation, InListValidation
+from pandas_schema.validation import MatchesPatternValidation
 import boto3
 from botocore.exceptions import ClientError
 from io import StringIO
-from pandas.util import hash_pandas_object
-import numpy
-from psycopg2.extensions import register_adapter, AsIs
 import logging
 import time
 import re
 from sqlalchemy import create_engine
-from sqlalchemy.types import String
-from sqlalchemy.types import NVARCHAR
 from sqlalchemy.types import Text
-from django.db import connection
+from django.conf import settings
 
-# Postgres Database Settings - local
-PG_HOST = os.getenv('FECFILE_DB_HOST', 'localhost')
-PG_PORT = os.getenv('FECFILE_DB_PORT', '5432')
-PG_DATABASE = os.getenv('FECFILE_DB_NAME', 'postgres')
-PG_USER = os.getenv('FECFILE_DB_USERNAME', 'postgres')
 PG_PASSWORD = os.getenv('FECFILE_DB_PASSWORD', 'postgres')
 SQS_QUEUE_NAME = os.getenv('SQS_QUEUE_NAME')
 
-
 BACKEND_DB_HOST = os.getenv('BACKEND_DB_HOST')
 BACKEND_DB_PORT = os.getenv('BACKEND_DB_PORT')
 BACKEND_DB_NAME = os.getenv('BACKEND_DB_NAME')
@@ -57,7 +45,7 @@ def save_data_from_excel_to_db(data):
     postgreSQLTable = 'ref_forms_scheds_format_specs'
     try:
         # "postgres://PG_USER:PG_PASSWORD@PG_HOST:PG_PORT/PG_DATABASE"
-        connectionstring = "postgres://" + PG_USER + ":" + PG_PASSWORD + "@" + PG_HOST + ":" + PG_PORT + "/" + PG_DATABASE
+        connectionstring = settings.DATABASE_URL
         engine = create_engine(connectionstring, pool_recycle=3600)
         postgreSQLConnection = engine.connect()
         data.to_sql(postgreSQLTable, postgreSQLConnection, if_exists='append', index=False, dtype={'AUTO-GENERATE': Text})
@@ -176,11 +164,7 @@ def schema_validation(dataframe, schema, bktname, key, errorfilename):
 
 def build_schemas(formname, sched, trans_type):
     try:
-        connection = psycopg2.connect(user=PG_USER,
-                                      password=PG_PASSWORD,
-                                      host=PG_HOST,
-                                      port=PG_PORT,
-                                      database=PG_DATABASE)
+        connection = psycopg2.connect(settings.DATABASE_URL)
         cursor = connection.cursor()
         cursor.execute("SELECT rfsfs.formname, rfsfs.schedname, rfsfs.transaction_type, rfsfs.field_description, rfsfs.type, rfsfs.required FROM public.ref_forms_scheds_format_specs rfsfs WHERE rfsfs.formname  = %s AND rfsfs.schedname = %s AND rfsfs.transaction_type = %s and rfsfs.type IS NOT NULL", (formname, sched, trans_type))
         format_specs = cursor.fetchall()
@@ -415,13 +399,7 @@ def check_data_processed(md5, fecfilename):
             WHERE tfd.fec_file_name = %s
             ORDER BY create_Date DESC   limit 1;
         '''
-        conn = psycopg2.connect(
-            user=PG_USER,
-            password=PG_PASSWORD,
-            host=PG_HOST,
-            port=PG_PORT,
-            database=PG_DATABASE
-        )
+        conn = psycopg2.connect(settings.DATABASE_URL)
         cur = conn.cursor()
         cur.execute(selectsql, (fecfilename,))
         if cur.rowcount == 1:
@@ -450,11 +428,7 @@ def load_transactions_from_temp_perm_tables(fecfilename):
     try:
         res = ''
         selectsql = '''import_schedules'''
-        conn = psycopg2.connect(user=PG_USER,
-                                password=PG_PASSWORD,
-                                host=PG_HOST,
-                                port=PG_PORT,
-                                database=PG_DATABASE)
+        conn = psycopg2.connect(settings.DATABASE_URL)
         cur = conn.cursor()
         cur.callproc(selectsql, (fecfilename,))
         if cur.rowcount == 1: