Merge pull request #25 from fecgov/schedule-a-model

Schedule A model

.circleci/config.yml

@@ -115,7 +115,7 @@ jobs:
       - run:
           name: Run tests
           # Use built-in Django test module
-          command: coverage run --source='.' manage.py test --keep
+          command: coverage run --source='.' --rcfile=.coveragerc manage.py test --keep
           working_directory: ~/project/django-backend/
 
       - run:

django-backend/.coveragerc

@@ -0,0 +1,6 @@
+# .coveragerc to control coverage.py
+# https://coverage.readthedocs.io/en/coverage-4.3.3/source.html#execution
+[run]
+omit =
+    # Excluding dev scripts from code coverage
+    ./scripts/json_schema_to_django_model.py

django-backend/Dockerfile

@@ -3,8 +3,6 @@ ENV PYTHONUNBUFFERED=1
 
 RUN mkdir /opt/nxg_fec
 WORKDIR /opt/nxg_fec
-# MacOS has trouble with these installs unless they're pulled out and run with these parameters
-RUN pip3 install Cython && pip3 install --no-binary :all: --no-use-pep517 numpy==1.17.2 && pip3 install pandas==0.25.1
 ADD requirements.txt /opt/nxg_fec/
 RUN pip3 install -r requirements.txt
 

django-backend/fecfiler/scha_transactions/admin.py

@@ -0,0 +1 @@
+# Register your models here.

django-backend/fecfiler/scha_transactions/apps.py

@@ -0,0 +1,9 @@
+from django.apps import AppConfig
+
+
+class SchedAConfig(AppConfig):
+    name = 'fecfiler.scha_transactions'
+
+    def ready(self):
+        # Implicitly connect a signal handlers decorated with @receiver.
+        from . import signals # noqa

django-backend/fecfiler/scha_transactions/migrations/0001_initial.py

@@ -0,0 +1,71 @@
+# Generated by Django 3.2.11 on 2022-02-03 01:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SchATransaction',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('form_type', models.CharField(max_length=8)),
+                ('filer_committee_id_number', models.CharField(max_length=9)),
+                ('transaction_id', models.CharField(max_length=20)),
+                ('back_reference_tran_id_number', models.CharField(blank=True, max_length=20, null=True)),
+                ('back_reference_sched_name', models.CharField(blank=True, max_length=8, null=True)),
+                ('entity_type', models.CharField(max_length=3)),
+                ('contributor_organization_name', models.CharField(max_length=200)),
+                ('contributor_last_name', models.CharField(max_length=30)),
+                ('contributor_first_name', models.CharField(max_length=20)),
+                ('contributor_middle_name', models.CharField(blank=True, max_length=20, null=True)),
+                ('contributor_prefix', models.CharField(blank=True, max_length=10, null=True)),
+                ('contributor_suffix', models.CharField(blank=True, max_length=10, null=True)),
+                ('contributor_street_1', models.CharField(blank=True, max_length=34, null=True)),
+                ('contributor_street_2', models.CharField(blank=True, max_length=34, null=True)),
+                ('contributor_city', models.CharField(blank=True, max_length=30, null=True)),
+                ('contributor_state', models.CharField(blank=True, max_length=2, null=True)),
+                ('contributor_zip', models.CharField(blank=True, max_length=9, null=True)),
+                ('election_code', models.CharField(blank=True, max_length=5, null=True)),
+                ('election_other_description', models.CharField(blank=True, max_length=20, null=True)),
+                ('contribution_date', models.IntegerField(blank=True, null=True)),
+                ('contribution_amount', models.IntegerField(blank=True, null=True)),
+                ('contribution_aggregate', models.IntegerField(blank=True, null=True)),
+                ('contribution_purpose_descrip', models.CharField(blank=True, max_length=100, null=True)),
+                ('contributor_employer', models.CharField(blank=True, max_length=38, null=True)),
+                ('contributor_occupation', models.CharField(blank=True, max_length=38, null=True)),
+                ('donor_committee_fec_id', models.CharField(blank=True, max_length=9, null=True)),
+                ('donor_committee_name', models.CharField(blank=True, max_length=200, null=True)),
+                ('donor_candidate_fec_id', models.CharField(blank=True, max_length=9, null=True)),
+                ('donor_candidate_last_name', models.CharField(blank=True, max_length=30, null=True)),
+                ('donor_candidate_first_name', models.CharField(blank=True, max_length=20, null=True)),
+                ('donor_candidate_middle_name', models.CharField(blank=True, max_length=20, null=True)),
+                ('donor_candidate_prefix', models.CharField(blank=True, max_length=10, null=True)),
+                ('donor_candidate_suffix', models.CharField(blank=True, max_length=10, null=True)),
+                ('donor_candidate_office', models.CharField(blank=True, max_length=1, null=True)),
+                ('donor_candidate_state', models.CharField(blank=True, max_length=2, null=True)),
+                ('donor_candidate_district', models.IntegerField(blank=True, null=True)),
+                ('conduit_name', models.CharField(blank=True, max_length=200, null=True)),
+                ('conduit_street1', models.CharField(blank=True, max_length=34, null=True)),
+                ('conduit_street2', models.CharField(blank=True, max_length=34, null=True)),
+                ('conduit_city', models.CharField(blank=True, max_length=30, null=True)),
+                ('conduit_state', models.CharField(blank=True, max_length=2, null=True)),
+                ('conduit_zip', models.CharField(blank=True, max_length=9, null=True)),
+                ('memo_code', models.CharField(blank=True, max_length=1, null=True)),
+                ('memo_text_description', models.CharField(blank=True, max_length=100, null=True)),
+                ('reference_to_si_or_sl_system_code_that_identifies_the_account', models.CharField(blank=True, max_length=9, null=True)),
+                ('transaction_type_identifier', models.CharField(blank=True, max_length=12, null=True)),
+                ('created', models.DateField(auto_now_add=True)),
+                ('updated', models.DateField(auto_now=True)),
+            ],
+            options={
+                'db_table': 'scha_transactions',
+            },
+        ),
+    ]

django-backend/fecfiler/scha_transactions/models.py

@@ -0,0 +1,58 @@
+from django.db import models
+import logging
+logger = logging.getLogger(__name__)
+
+
+class SchATransaction(models.Model):
+    """Generated model from json schema"""
+    form_type = models.CharField(null=False, blank=False, max_length=8)
+    filer_committee_id_number = models.CharField(null=False, blank=False, max_length=9)
+    transaction_id = models.CharField(null=False, blank=False, max_length=20)
+    back_reference_tran_id_number = models.CharField(null=True, blank=True, max_length=20)
+    back_reference_sched_name = models.CharField(null=True, blank=True, max_length=8)
+    entity_type = models.CharField(null=False, blank=False, max_length=3)
+    contributor_organization_name = models.CharField(null=False, blank=False, max_length=200)
+    contributor_last_name = models.CharField(null=False, blank=False, max_length=30)
+    contributor_first_name = models.CharField(null=False, blank=False, max_length=20)
+    contributor_middle_name = models.CharField(null=True, blank=True, max_length=20)
+    contributor_prefix = models.CharField(null=True, blank=True, max_length=10)
+    contributor_suffix = models.CharField(null=True, blank=True, max_length=10)
+    contributor_street_1 = models.CharField(null=True, blank=True, max_length=34)
+    contributor_street_2 = models.CharField(null=True, blank=True, max_length=34)
+    contributor_city = models.CharField(null=True, blank=True, max_length=30)
+    contributor_state = models.CharField(null=True, blank=True, max_length=2)
+    contributor_zip = models.CharField(null=True, blank=True, max_length=9)
+    election_code = models.CharField(null=True, blank=True, max_length=5)
+    election_other_description = models.CharField(null=True, blank=True, max_length=20)
+    contribution_date = models.IntegerField(null=True, blank=True)
+    contribution_amount = models.IntegerField(null=True, blank=True)
+    contribution_aggregate = models.IntegerField(null=True, blank=True)
+    contribution_purpose_descrip = models.CharField(null=True, blank=True, max_length=100)
+    contributor_employer = models.CharField(null=True, blank=True, max_length=38)
+    contributor_occupation = models.CharField(null=True, blank=True, max_length=38)
+    donor_committee_fec_id = models.CharField(null=True, blank=True, max_length=9)
+    donor_committee_name = models.CharField(null=True, blank=True, max_length=200)
+    donor_candidate_fec_id = models.CharField(null=True, blank=True, max_length=9)
+    donor_candidate_last_name = models.CharField(null=True, blank=True, max_length=30)
+    donor_candidate_first_name = models.CharField(null=True, blank=True, max_length=20)
+    donor_candidate_middle_name = models.CharField(null=True, blank=True, max_length=20)
+    donor_candidate_prefix = models.CharField(null=True, blank=True, max_length=10)
+    donor_candidate_suffix = models.CharField(null=True, blank=True, max_length=10)
+    donor_candidate_office = models.CharField(null=True, blank=True, max_length=1)
+    donor_candidate_state = models.CharField(null=True, blank=True, max_length=2)
+    donor_candidate_district = models.IntegerField(null=True, blank=True)
+    conduit_name = models.CharField(null=True, blank=True, max_length=200)
+    conduit_street1 = models.CharField(null=True, blank=True, max_length=34)
+    conduit_street2 = models.CharField(null=True, blank=True, max_length=34)
+    conduit_city = models.CharField(null=True, blank=True, max_length=30)
+    conduit_state = models.CharField(null=True, blank=True, max_length=2)
+    conduit_zip = models.CharField(null=True, blank=True, max_length=9)
+    memo_code = models.CharField(null=True, blank=True, max_length=1)
+    memo_text_description = models.CharField(null=True, blank=True, max_length=100)
+    reference_to_si_or_sl_system_code_that_identifies_the_account = models.CharField(null=True, blank=True, max_length=9)
+    transaction_type_identifier = models.CharField(null=True, blank=True, max_length=12)
+    created = models.DateField(auto_now_add=True)
+    updated = models.DateField(auto_now=True)
+
+    class Meta:
+        db_table = 'scha_transactions'

django-backend/fecfiler/scha_transactions/signals.py

@@ -0,0 +1,27 @@
+"""Logs Schedule A Transaction events
+
+We use signals to log deletes rather than overwriting delete()
+to handle bulk delete cases
+https://docs.djangoproject.com/en/dev/topics/db/models/#overriding-predefined-model-methods
+
+We use signals to log saves to be consistent with delete logging
+"""
+from django.db.models.signals import post_save, post_delete
+from django.dispatch import receiver
+from .models import SchATransaction
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+@receiver(post_save, sender=SchATransaction)
+def log_post_save(sender, instance, created, **kwargs):
+    action = 'created' if created else 'updated'
+    logger.info('Schedule A Transaction: %s was %s',
+                instance.transaction_id, action)
+
+
+@receiver(post_delete, sender=SchATransaction)
+def log_post_delete(sender, instance, **kwargs):
+    logger.info('Schedule A Transaction: %s was deleted',
+                instance.transaction_id)

django-backend/fecfiler/scha_transactions/tests.py

@@ -0,0 +1,54 @@
+from django.test import TestCase
+from django.core.exceptions import ValidationError
+from .models import SchATransaction
+
+
+class SchATransactionTestCase(TestCase):
+    """ Test module for inserting a sched_a item"""
+
+    def setUp(self):
+        self.sa_trans = SchATransaction(
+            form_type="SA11AI",
+            filer_committee_id_number="C00123456",
+            transaction_id="A56123456789-1234",
+            entity_type="IND",
+            contributor_organization_name="John Smith & Co.",
+            contributor_first_name="John",
+            contributor_last_name="Smith"
+        )
+
+        self.bad_trans = SchATransaction(
+            form_type="SA11AI",
+            filer_committee_id_number="C00123456",
+            transaction_id="A56123456789-4567",
+            entity_type="IND",
+            contributor_organization_name="Some Org",
+            contributor_first_name="John",
+        )
+
+        self.trans_to_del = SchATransaction(
+            form_type="SA11AI",
+            filer_committee_id_number="C00123456",
+            transaction_id="A56123456789-del",
+            entity_type="IND",
+            contributor_organization_name="Group",
+            contributor_first_name="John",
+            contributor_last_name="Smith"
+        )
+
+    def test_full_clean(self):
+        self.sa_trans.full_clean()
+        self.assertRaises(ValidationError, self.bad_trans.full_clean)
+
+    def test_save(self):
+        self.sa_trans.save()
+        trans = SchATransaction.objects.get(transaction_id="A56123456789-1234")
+        self.assertIsInstance(trans, SchATransaction)
+        self.assertEquals(trans.transaction_id, "A56123456789-1234")
+
+    def test_delete(self):
+        self.trans_to_del.save()
+        hit = SchATransaction.objects.get(transaction_id="A56123456789-del")
+        self.assertEquals(hit.transaction_id, "A56123456789-del")
+        hit.delete()
+        self.assertRaises(SchATransaction.DoesNotExist, SchATransaction.objects.get, transaction_id="A56123456789-del")

django-backend/fecfiler/sched_C/models.py

@@ -81,7 +81,7 @@ class SchedC(models.Model):
     lender_cand_office = models.CharField(max_length=1, blank=True, null=True)
     lender_cand_state = models.CharField(max_length=2, blank=True, null=True)
     lender_cand_district = models.DecimalField(
-        max_digits=65535, blank=True, null=True)
+        max_digits=12, blank=True, null=True, decimal_places=2)
     memo_code = models.CharField(max_length=1, blank=True, null=True)
     memo_text = models.CharField(max_length=100, blank=True, null=True)
     delete_ind = models.CharField(max_length=1, blank=True, null=True)

django-backend/fecfiler/settings.py

@@ -70,8 +70,6 @@
 # Application definition
 
 INSTALLED_APPS = [
-    'djangocms_admin_style',
-    'admin_shortcuts',
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',
@@ -87,6 +85,7 @@
     'fecfiler.forms',
     'db_file_storage',
 
+    'fecfiler.scha_transactions',
     'fecfiler.core',
     # 'fecfiler.form3x',
     'fecfiler.sched_A',

django-backend/requirements.txt

@@ -24,7 +24,7 @@ django-db-file-storage==0.5.3
 django-rest-swagger==2.2.0
 django-s3-upload==0.2.1
 django-storages==1.12.3
-djangocms-admin-style==1.2.5
+djangocms-admin-style
 djangorestframework==3.13.1
 djangorestframework-jwt==1.11.0
 djangorestframework-sso==0.3.2
@@ -74,7 +74,7 @@ wcwidth==0.1.7
 fuzzywuzzy==0.17.0
 python-levenshtein==0.12.2
 numpy==1.21.5
-pandas==0.25.1
+pandas==1.3.5
 pandas-schema==0.3.5
 django-otp==0.9.3
 callfire-api-client-python==0.0.9