diff --git a/.circleci/config.yml b/.circleci/config.yml index 98af3ce4e4..0431855b3a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -115,7 +115,7 @@ jobs: - run: name: Run tests # Use built-in Django test module - command: coverage run --source='.' manage.py test --keep + command: coverage run --source='.' --rcfile=.coveragerc manage.py test --keep working_directory: ~/project/django-backend/ - run: diff --git a/django-backend/.coveragerc b/django-backend/.coveragerc new file mode 100644 index 0000000000..32d237e785 --- /dev/null +++ b/django-backend/.coveragerc @@ -0,0 +1,6 @@ +# .coveragerc to control coverage.py +# https://coverage.readthedocs.io/en/coverage-4.3.3/source.html#execution +[run] +omit = + # Excluding dev scripts from code coverage + ./scripts/json_schema_to_django_model.py \ No newline at end of file diff --git a/django-backend/Dockerfile b/django-backend/Dockerfile index e005f612af..a6ec0c7551 100644 --- a/django-backend/Dockerfile +++ b/django-backend/Dockerfile @@ -3,8 +3,6 @@ ENV PYTHONUNBUFFERED=1 RUN mkdir /opt/nxg_fec WORKDIR /opt/nxg_fec -# MacOS has trouble with these installs unless they're pulled out and run with these parameters -RUN pip3 install Cython && pip3 install --no-binary :all: --no-use-pep517 numpy==1.17.2 && pip3 install pandas==0.25.1 ADD requirements.txt /opt/nxg_fec/ RUN pip3 install -r requirements.txt diff --git a/django-backend/fecfiler/scha_transactions/__init__.py b/django-backend/fecfiler/scha_transactions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django-backend/fecfiler/scha_transactions/admin.py b/django-backend/fecfiler/scha_transactions/admin.py new file mode 100644 index 0000000000..846f6b4061 --- /dev/null +++ b/django-backend/fecfiler/scha_transactions/admin.py @@ -0,0 +1 @@ +# Register your models here. diff --git a/django-backend/fecfiler/scha_transactions/apps.py b/django-backend/fecfiler/scha_transactions/apps.py new file mode 100644 index 0000000000..493d85a94a --- /dev/null +++ b/django-backend/fecfiler/scha_transactions/apps.py @@ -0,0 +1,9 @@ +from django.apps import AppConfig + + +class SchedAConfig(AppConfig): + name = 'fecfiler.scha_transactions' + + def ready(self): + # Implicitly connect a signal handlers decorated with @receiver. + from . import signals # noqa diff --git a/django-backend/fecfiler/scha_transactions/migrations/0001_initial.py b/django-backend/fecfiler/scha_transactions/migrations/0001_initial.py new file mode 100644 index 0000000000..492b7a6ff8 --- /dev/null +++ b/django-backend/fecfiler/scha_transactions/migrations/0001_initial.py @@ -0,0 +1,71 @@ +# Generated by Django 3.2.11 on 2022-02-03 01:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='SchATransaction', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('form_type', models.CharField(max_length=8)), + ('filer_committee_id_number', models.CharField(max_length=9)), + ('transaction_id', models.CharField(max_length=20)), + ('back_reference_tran_id_number', models.CharField(blank=True, max_length=20, null=True)), + ('back_reference_sched_name', models.CharField(blank=True, max_length=8, null=True)), + ('entity_type', models.CharField(max_length=3)), + ('contributor_organization_name', models.CharField(max_length=200)), + ('contributor_last_name', models.CharField(max_length=30)), + ('contributor_first_name', models.CharField(max_length=20)), + ('contributor_middle_name', models.CharField(blank=True, max_length=20, null=True)), + ('contributor_prefix', models.CharField(blank=True, max_length=10, null=True)), + ('contributor_suffix', models.CharField(blank=True, max_length=10, null=True)), + ('contributor_street_1', models.CharField(blank=True, max_length=34, null=True)), + ('contributor_street_2', models.CharField(blank=True, max_length=34, null=True)), + ('contributor_city', models.CharField(blank=True, max_length=30, null=True)), + ('contributor_state', models.CharField(blank=True, max_length=2, null=True)), + ('contributor_zip', models.CharField(blank=True, max_length=9, null=True)), + ('election_code', models.CharField(blank=True, max_length=5, null=True)), + ('election_other_description', models.CharField(blank=True, max_length=20, null=True)), + ('contribution_date', models.IntegerField(blank=True, null=True)), + ('contribution_amount', models.IntegerField(blank=True, null=True)), + ('contribution_aggregate', models.IntegerField(blank=True, null=True)), + ('contribution_purpose_descrip', models.CharField(blank=True, max_length=100, null=True)), + ('contributor_employer', models.CharField(blank=True, max_length=38, null=True)), + ('contributor_occupation', models.CharField(blank=True, max_length=38, null=True)), + ('donor_committee_fec_id', models.CharField(blank=True, max_length=9, null=True)), + ('donor_committee_name', models.CharField(blank=True, max_length=200, null=True)), + ('donor_candidate_fec_id', models.CharField(blank=True, max_length=9, null=True)), + ('donor_candidate_last_name', models.CharField(blank=True, max_length=30, null=True)), + ('donor_candidate_first_name', models.CharField(blank=True, max_length=20, null=True)), + ('donor_candidate_middle_name', models.CharField(blank=True, max_length=20, null=True)), + ('donor_candidate_prefix', models.CharField(blank=True, max_length=10, null=True)), + ('donor_candidate_suffix', models.CharField(blank=True, max_length=10, null=True)), + ('donor_candidate_office', models.CharField(blank=True, max_length=1, null=True)), + ('donor_candidate_state', models.CharField(blank=True, max_length=2, null=True)), + ('donor_candidate_district', models.IntegerField(blank=True, null=True)), + ('conduit_name', models.CharField(blank=True, max_length=200, null=True)), + ('conduit_street1', models.CharField(blank=True, max_length=34, null=True)), + ('conduit_street2', models.CharField(blank=True, max_length=34, null=True)), + ('conduit_city', models.CharField(blank=True, max_length=30, null=True)), + ('conduit_state', models.CharField(blank=True, max_length=2, null=True)), + ('conduit_zip', models.CharField(blank=True, max_length=9, null=True)), + ('memo_code', models.CharField(blank=True, max_length=1, null=True)), + ('memo_text_description', models.CharField(blank=True, max_length=100, null=True)), + ('reference_to_si_or_sl_system_code_that_identifies_the_account', models.CharField(blank=True, max_length=9, null=True)), + ('transaction_type_identifier', models.CharField(blank=True, max_length=12, null=True)), + ('created', models.DateField(auto_now_add=True)), + ('updated', models.DateField(auto_now=True)), + ], + options={ + 'db_table': 'scha_transactions', + }, + ), + ] diff --git a/django-backend/fecfiler/scha_transactions/migrations/__init__.py b/django-backend/fecfiler/scha_transactions/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django-backend/fecfiler/scha_transactions/models.py b/django-backend/fecfiler/scha_transactions/models.py new file mode 100644 index 0000000000..54f39bbcdb --- /dev/null +++ b/django-backend/fecfiler/scha_transactions/models.py @@ -0,0 +1,58 @@ +from django.db import models +import logging +logger = logging.getLogger(__name__) + + +class SchATransaction(models.Model): + """Generated model from json schema""" + form_type = models.CharField(null=False, blank=False, max_length=8) + filer_committee_id_number = models.CharField(null=False, blank=False, max_length=9) + transaction_id = models.CharField(null=False, blank=False, max_length=20) + back_reference_tran_id_number = models.CharField(null=True, blank=True, max_length=20) + back_reference_sched_name = models.CharField(null=True, blank=True, max_length=8) + entity_type = models.CharField(null=False, blank=False, max_length=3) + contributor_organization_name = models.CharField(null=False, blank=False, max_length=200) + contributor_last_name = models.CharField(null=False, blank=False, max_length=30) + contributor_first_name = models.CharField(null=False, blank=False, max_length=20) + contributor_middle_name = models.CharField(null=True, blank=True, max_length=20) + contributor_prefix = models.CharField(null=True, blank=True, max_length=10) + contributor_suffix = models.CharField(null=True, blank=True, max_length=10) + contributor_street_1 = models.CharField(null=True, blank=True, max_length=34) + contributor_street_2 = models.CharField(null=True, blank=True, max_length=34) + contributor_city = models.CharField(null=True, blank=True, max_length=30) + contributor_state = models.CharField(null=True, blank=True, max_length=2) + contributor_zip = models.CharField(null=True, blank=True, max_length=9) + election_code = models.CharField(null=True, blank=True, max_length=5) + election_other_description = models.CharField(null=True, blank=True, max_length=20) + contribution_date = models.IntegerField(null=True, blank=True) + contribution_amount = models.IntegerField(null=True, blank=True) + contribution_aggregate = models.IntegerField(null=True, blank=True) + contribution_purpose_descrip = models.CharField(null=True, blank=True, max_length=100) + contributor_employer = models.CharField(null=True, blank=True, max_length=38) + contributor_occupation = models.CharField(null=True, blank=True, max_length=38) + donor_committee_fec_id = models.CharField(null=True, blank=True, max_length=9) + donor_committee_name = models.CharField(null=True, blank=True, max_length=200) + donor_candidate_fec_id = models.CharField(null=True, blank=True, max_length=9) + donor_candidate_last_name = models.CharField(null=True, blank=True, max_length=30) + donor_candidate_first_name = models.CharField(null=True, blank=True, max_length=20) + donor_candidate_middle_name = models.CharField(null=True, blank=True, max_length=20) + donor_candidate_prefix = models.CharField(null=True, blank=True, max_length=10) + donor_candidate_suffix = models.CharField(null=True, blank=True, max_length=10) + donor_candidate_office = models.CharField(null=True, blank=True, max_length=1) + donor_candidate_state = models.CharField(null=True, blank=True, max_length=2) + donor_candidate_district = models.IntegerField(null=True, blank=True) + conduit_name = models.CharField(null=True, blank=True, max_length=200) + conduit_street1 = models.CharField(null=True, blank=True, max_length=34) + conduit_street2 = models.CharField(null=True, blank=True, max_length=34) + conduit_city = models.CharField(null=True, blank=True, max_length=30) + conduit_state = models.CharField(null=True, blank=True, max_length=2) + conduit_zip = models.CharField(null=True, blank=True, max_length=9) + memo_code = models.CharField(null=True, blank=True, max_length=1) + memo_text_description = models.CharField(null=True, blank=True, max_length=100) + reference_to_si_or_sl_system_code_that_identifies_the_account = models.CharField(null=True, blank=True, max_length=9) + transaction_type_identifier = models.CharField(null=True, blank=True, max_length=12) + created = models.DateField(auto_now_add=True) + updated = models.DateField(auto_now=True) + + class Meta: + db_table = 'scha_transactions' diff --git a/django-backend/fecfiler/scha_transactions/signals.py b/django-backend/fecfiler/scha_transactions/signals.py new file mode 100644 index 0000000000..a95c75839a --- /dev/null +++ b/django-backend/fecfiler/scha_transactions/signals.py @@ -0,0 +1,27 @@ +"""Logs Schedule A Transaction events + +We use signals to log deletes rather than overwriting delete() +to handle bulk delete cases +https://docs.djangoproject.com/en/dev/topics/db/models/#overriding-predefined-model-methods + +We use signals to log saves to be consistent with delete logging +""" +from django.db.models.signals import post_save, post_delete +from django.dispatch import receiver +from .models import SchATransaction +import logging + +logger = logging.getLogger(__name__) + + +@receiver(post_save, sender=SchATransaction) +def log_post_save(sender, instance, created, **kwargs): + action = 'created' if created else 'updated' + logger.info('Schedule A Transaction: %s was %s', + instance.transaction_id, action) + + +@receiver(post_delete, sender=SchATransaction) +def log_post_delete(sender, instance, **kwargs): + logger.info('Schedule A Transaction: %s was deleted', + instance.transaction_id) diff --git a/django-backend/fecfiler/scha_transactions/tests.py b/django-backend/fecfiler/scha_transactions/tests.py new file mode 100644 index 0000000000..e3bd4fe7c1 --- /dev/null +++ b/django-backend/fecfiler/scha_transactions/tests.py @@ -0,0 +1,54 @@ +from django.test import TestCase +from django.core.exceptions import ValidationError +from .models import SchATransaction + + +class SchATransactionTestCase(TestCase): + """ Test module for inserting a sched_a item""" + + def setUp(self): + self.sa_trans = SchATransaction( + form_type="SA11AI", + filer_committee_id_number="C00123456", + transaction_id="A56123456789-1234", + entity_type="IND", + contributor_organization_name="John Smith & Co.", + contributor_first_name="John", + contributor_last_name="Smith" + ) + + self.bad_trans = SchATransaction( + form_type="SA11AI", + filer_committee_id_number="C00123456", + transaction_id="A56123456789-4567", + entity_type="IND", + contributor_organization_name="Some Org", + contributor_first_name="John", + ) + + self.trans_to_del = SchATransaction( + form_type="SA11AI", + filer_committee_id_number="C00123456", + transaction_id="A56123456789-del", + entity_type="IND", + contributor_organization_name="Group", + contributor_first_name="John", + contributor_last_name="Smith" + ) + + def test_full_clean(self): + self.sa_trans.full_clean() + self.assertRaises(ValidationError, self.bad_trans.full_clean) + + def test_save(self): + self.sa_trans.save() + trans = SchATransaction.objects.get(transaction_id="A56123456789-1234") + self.assertIsInstance(trans, SchATransaction) + self.assertEquals(trans.transaction_id, "A56123456789-1234") + + def test_delete(self): + self.trans_to_del.save() + hit = SchATransaction.objects.get(transaction_id="A56123456789-del") + self.assertEquals(hit.transaction_id, "A56123456789-del") + hit.delete() + self.assertRaises(SchATransaction.DoesNotExist, SchATransaction.objects.get, transaction_id="A56123456789-del") diff --git a/django-backend/fecfiler/scha_transactions/urls.py b/django-backend/fecfiler/scha_transactions/urls.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django-backend/fecfiler/scha_transactions/views.py b/django-backend/fecfiler/scha_transactions/views.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django-backend/fecfiler/sched_C/models.py b/django-backend/fecfiler/sched_C/models.py index 4fbbeca85b..1a26847d90 100644 --- a/django-backend/fecfiler/sched_C/models.py +++ b/django-backend/fecfiler/sched_C/models.py @@ -81,7 +81,7 @@ class SchedC(models.Model): lender_cand_office = models.CharField(max_length=1, blank=True, null=True) lender_cand_state = models.CharField(max_length=2, blank=True, null=True) lender_cand_district = models.DecimalField( - max_digits=65535, blank=True, null=True) + max_digits=12, blank=True, null=True, decimal_places=2) memo_code = models.CharField(max_length=1, blank=True, null=True) memo_text = models.CharField(max_length=100, blank=True, null=True) delete_ind = models.CharField(max_length=1, blank=True, null=True) diff --git a/django-backend/fecfiler/settings.py b/django-backend/fecfiler/settings.py index 3d9d11963b..8a76d58aad 100644 --- a/django-backend/fecfiler/settings.py +++ b/django-backend/fecfiler/settings.py @@ -70,8 +70,6 @@ # Application definition INSTALLED_APPS = [ - 'djangocms_admin_style', - 'admin_shortcuts', 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', @@ -87,6 +85,7 @@ 'fecfiler.forms', 'db_file_storage', + 'fecfiler.scha_transactions', 'fecfiler.core', # 'fecfiler.form3x', 'fecfiler.sched_A', diff --git a/django-backend/requirements.txt b/django-backend/requirements.txt index e3ef2491c5..99f1e8cac8 100644 --- a/django-backend/requirements.txt +++ b/django-backend/requirements.txt @@ -24,7 +24,7 @@ django-db-file-storage==0.5.3 django-rest-swagger==2.2.0 django-s3-upload==0.2.1 django-storages==1.12.3 -djangocms-admin-style==1.2.5 +djangocms-admin-style djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 djangorestframework-sso==0.3.2 @@ -74,7 +74,7 @@ wcwidth==0.1.7 fuzzywuzzy==0.17.0 python-levenshtein==0.12.2 numpy==1.21.5 -pandas==0.25.1 +pandas==1.3.5 pandas-schema==0.3.5 django-otp==0.9.3 callfire-api-client-python==0.0.9 diff --git a/django-backend/scripts/json_schema_to_django_model.py b/django-backend/scripts/json_schema_to_django_model.py new file mode 100644 index 0000000000..2e22f740f7 --- /dev/null +++ b/django-backend/scripts/json_schema_to_django_model.py @@ -0,0 +1,205 @@ +"""Converts JSON schema to a workable django model + +This is a utility script to create *starter* django model from +a JSON schema file + +The JSON schema standard can be found here: + +http://json-schema.org/ + +Note: Schema properties prefixed with "fec_" are not part of the JSON schema +standard and are specific to the FEC data. +""" +import json +import argparse +import logging +import os + + +def determine_model_name(model_id=None, filename=None): + """ + Get the model name + :param model_id: str, model id + :param filename: str, filename + :return: str, model name + """ + model_name = '' + if model_id: + try: + model_name = model_id.split('/')[-1].replace('.json', '') + except Exception as e: + logging.exception("Unhandled exception {}".format(e)) + + if not model_name and filename: + filename = filename.strip(os.sep) + model_name = filename.split(os.sep)[-1] + model_name = model_name.split('.')[0] + + return model_name.capitalize() or 'UnknownModel' + + +def get_required_string(key_name, required_fields, field_type='string', is_pk_field=False): + """ + Gets the required portion of model field + :param key_name: + :param required_fields: + :return: str, required model string + """ + if is_pk_field: + return 'primary_key=True' + + if key_name in required_fields: + return 'null=False, blank=False' + return 'null=True, blank=True' + + +def parse_model(json_model): # noqa + + # Make sure not list, but object + if json_model['type'] != 'object': + print("Model type has to be object to convert to model, got {}".format(json_model['type'])) + + if 'oneOf' in json_model: + print("Optional required fields detected: {}".format(json_model['oneOf'])) + + # Default model string + model_str = "\nfrom django import models\nfrom django.models import json\n\n" + + model_name = determine_model_name(json_model.get('id'), args.filename) + model_str += "class {}(models.Model):\n".format(model_name) + model_str += ' """Generated model from json schema"""\n' + print("Model name is {}".format(model_name)) + + if 'title' in json_model: + print("Title of model is {}".format(json_model['title'])) + + if 'description' in json_model: + print("Description of model is {}".format(json_model['description'])) + + required_fields = [] + if 'required' in json_model: + required_fields = json_model['required'] + + for key_name, key_attributes in json_model['properties'].items(): + if key_name.endswith('_id') and key_name != '_id': + print("WARNING: Possible ForeignKey {}".format(key_name)) + + if key_attributes['type'] == 'null': + print("ERROR: Unsupported type null, skipping for field {}".format(key_name)) + + # PK field + is_pk_field = False + if key_name in ['id', '_id']: + is_pk_field = True + + # If required field + required_str = get_required_string(key_name, required_fields, key_attributes['type'], is_pk_field) + field_str = '' + + # String choice field, enum + if key_attributes['type'] == 'string' and 'enum' in key_attributes: + if not key_attributes['enum']: + print("ERROR: Missing enum for enum choice field {}, skipping..".format(key_name)) + continue + + if len(key_attributes['enum']) == 1: + print("WARNING: enum value with single choice for field {}, choice {}." + "".format(key_name, key_attributes['enum'])) + continue + + # Max length find + max_length = 255 + for choice in key_attributes['enum']: + if len(choice) > 255: + max_length = len(choice) + + choices = tuple(set(zip(key_attributes['enum'], key_attributes['enum']))) + + field_str = " {} = models.CharField(choices={}, max_length={}, " \ + "default='{}', {})\n" \ + "".format(key_name, choices, max_length, key_attributes['enum'][0], required_str) + + # Date time field + elif key_attributes['type'] == 'string' and key_attributes.get('format') == 'date-time': + auto_now_add = False + editable = True + if key_name in ['created_on', 'modified_on']: + auto_now_add = True + editable = False + + field_str = " {} = models.DateTimeField(auto_now_add={}, editable={}, {})\n" \ + "".format(key_name, auto_now_add, editable, required_str) + + elif key_attributes['type'] == 'integer': + max_length = key_attributes.get('maxLength') + if max_length is not None: + field_str = " {} = models.IntegerField({}, max_length={})\n".format(key_name, required_str, max_length) + else: + field_str = " {} = models.IntegerField({})\n".format(key_name, required_str) + + elif key_attributes['type'] == 'string': + max_length = key_attributes.get('maxLength') + if max_length is not None: + field_str = " {} = models.CharField({}, max_length={})\n".format(key_name, required_str, max_length) + else: + field_str = " {} = models.TextField({})\n".format(key_name, required_str) + + elif key_attributes['type'] == 'number': + field_str = " {} = models.IntegerField({})\n".format(key_name, required_str) + + elif key_attributes['type'] == 'array': + field_str = " {} = json.JSONField(default=[], {})\n".format(key_name, required_str) + + elif key_attributes['type'] == 'object': + field_str = " {} = json.JSONField(default={{}}, {})\n".format(key_name, required_str) + + elif key_attributes['type'] == 'boolean': + field_str = " {} = models.BooleanField(default=False, {})\n".format(key_name, required_str) + + model_str += field_str + + # add created and updated fields + model_str += " created = models.DateField(auto_now_add=True)\n" + model_str += " updated = models.DateField(auto_now=True)\n" + model_str += "\n class Meta:\n db_table = '{}s'\n".format(model_name.lower()) + + return model_name, model_str + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('filename') + args = parser.parse_args() + filename = args.filename + + with open(filename) as f: + json_model = json.load(f) + + if ("S" == json_model['properties']['form_type']['examples'][0][0] + and "transaction_type_identifier" not in json_model['properties']): + print("We've detected that this schema is likely a schedule and yet it has no transaction_type_identifier field.") + print("Would you like us to add this field (y/N)?") + choice = input().lower() + add_tti = choice in ["y", "ye", "yes"] + + if add_tti: + json_model['properties']['transaction_type_identifier'] = { + "title": "TRANSACTION TYPE IDENTIFIER", + "description": "", + "type": "string", + "minLength": 0, + "maxLength": 12, + "pattern": "^[ A-z0-9]{0,12}$", + "examples": [ + "IK_PAC_REC" + ], + "fec_form_line": "0", + "fec_type": "A/N-12", + "fec_requiredErrorLevel": "X (error)" + } + + model_name, model_str = parse_model(json_model) + f = open(model_name + '.py', "w") + f.write(model_str) + f.close() + print('Done') diff --git a/django-backend/templates/javascripts.html b/django-backend/templates/javascripts.html index d718fa4ae9..d0d6e0e6c4 100644 --- a/django-backend/templates/javascripts.html +++ b/django-backend/templates/javascripts.html @@ -1,5 +1,5 @@ {% load compress %} -{% load staticfiles %} +{% load static %} {% compress js %} diff --git a/django-backend/templates/stylesheets.html b/django-backend/templates/stylesheets.html index ff908f0beb..3c68914351 100644 --- a/django-backend/templates/stylesheets.html +++ b/django-backend/templates/stylesheets.html @@ -1,5 +1,5 @@ {% load compress %} -{% load staticfiles %} +{% load static %} {% compress css %} diff --git a/sonar-project.properties b/sonar-project.properties index 708f3e7d81..feb9dc71cc 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -11,6 +11,8 @@ sonar.sources=django-backend # Encoding of the source code. Default is default system encoding #sonar.sourceEncoding=UTF-8 +# Exclude utility script from coverage +sonar.coverage.exclusions=**/json_schema_to_django_model.py sonar.python.coverage.reportPaths=coverage-reports/coverage.xml sonar.python.bandit.reportPaths=bandit.out sonar.python.flake8.reportPaths=flake8.out