Merge pull request #315 from GSA/feature/art_language_db_migration

 Adding art_language db table

.gitignore

@@ -137,7 +137,7 @@ $RECYCLE.BIN/
 
 ### Data ###
 *.json
-*.sql
+data/*.sql
 *.xml
 *.csv
 !html_tags.txt

alembic/env.py

@@ -19,7 +19,7 @@
 # since 'utils/' and 'alembic/' are siblings folders, we can import the
 # relative module using the append method of the sys.path module
 sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
-from fbo_scraper.db.db_utils import get_db_url
+from fbo_scraper.db.connection import get_db_url
 
 config.set_main_option("sqlalchemy.url", get_db_url())
 from fbo_scraper.db import db

alembic/versions/2e38f559933b_adding_art_language_table.py

@@ -0,0 +1,39 @@
+"""Adding art language table
+
+Revision ID: 2e38f559933b
+Revises: 78823e9293e9
+Create Date: 2024-06-25 14:55:27.913403
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.dialects.postgresql import JSONB
+
+# revision identifiers, used by Alembic.
+revision = '2e38f559933b'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('art_language',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('solicitation_id', sa.Integer(), nullable=True),
+    sa.Column('language', JSONB(), nullable=False),
+    sa.Column('createdAt', sa.DateTime(), nullable=False),
+    sa.Column('updatedAt', sa.DateTime(), nullable=True),
+    sa.ForeignKeyConstraint(['solicitation_id'], ['solicitations.id'], name=op.f('fk_art_language_solicitation_solicitations')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_art_language'))
+    )
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    op.drop_table('art_language')
+    # ### end Alembic commands ###

sql/migrations/add_art_language.sql

@@ -0,0 +1,15 @@
+BEGIN;
+
+CREATE TABLE art_language (
+    id SERIAL NOT NULL, 
+    solicitation_id INTEGER, 
+    language JSONB NOT NULL, 
+    "createdAt" TIMESTAMP WITHOUT TIME ZONE NOT NULL, 
+    "updatedAt" TIMESTAMP WITHOUT TIME ZONE, 
+    CONSTRAINT pk_art_language PRIMARY KEY (id), 
+    CONSTRAINT fk_art_language_solicitation_solicitations FOREIGN KEY(solicitation_id) REFERENCES solicitations (id)
+);
+
+INSERT INTO alembic_version (version_num) VALUES ('2e38f559933b') RETURNING alembic_version.version_num;
+
+COMMIT;

sql/migrations/local/local_matching_db.sql

@@ -0,0 +1,79 @@
+BEGIN;
+
+ALTER TABLE "Agencies" ALTER COLUMN "updatedAt" DROP NOT NULL;
+
+ALTER TABLE "Predictions" ADD COLUMN "eitLikelihood" JSONB;
+
+ALTER TABLE "Predictions" ADD COLUMN active BOOLEAN DEFAULT true;
+
+ALTER TABLE "Predictions" ALTER COLUMN title SET NOT NULL;
+
+ALTER TABLE "Predictions" ALTER COLUMN "solNum" SET NOT NULL;
+
+ALTER TABLE "Predictions" ALTER COLUMN "noticeType" SET NOT NULL;
+
+ALTER TABLE "Predictions" ALTER COLUMN "createdAt" SET NOT NULL;
+
+ALTER TABLE "Predictions" ALTER COLUMN history SET DEFAULT '[]'::jsonb;
+
+ALTER TABLE "Predictions" ADD CONSTRAINT "uq_Predictions_solNum" UNIQUE ("solNum");
+
+ALTER TABLE "Predictions" DROP COLUMN feedback;
+
+ALTER TABLE "Predictions" DROP COLUMN category_list;
+
+ALTER TABLE "Surveys" ALTER COLUMN "updatedAt" DROP NOT NULL;
+
+ALTER TABLE "Users" ALTER COLUMN "updatedAt" DROP NOT NULL;
+
+ALTER TABLE agency_alias ALTER COLUMN agency_id SET NOT NULL;
+
+ALTER TABLE agency_alias ALTER COLUMN "createdAt" SET NOT NULL;
+
+ALTER TABLE agency_alias ALTER COLUMN "createdAt" SET DEFAULT now();
+
+ALTER TABLE agency_alias ALTER COLUMN "updatedAt" DROP NOT NULL;
+
+ALTER TABLE notice_type ADD COLUMN "createdAt" TIMESTAMP WITHOUT TIME ZONE NOT NULL;
+
+ALTER TABLE notice_type ADD COLUMN "updatedAt" TIMESTAMP WITHOUT TIME ZONE;
+
+ALTER TABLE survey_responses ALTER COLUMN "createdAt" SET NOT NULL;
+
+CREATE INDEX "ix_survey_responses_solNum" ON survey_responses ("solNum");
+
+ALTER TABLE survey_responses_archive ALTER COLUMN "createdAt" SET NOT NULL;
+
+ALTER TABLE survey_responses_archive ALTER COLUMN "createdAt" SET DEFAULT now();
+
+ALTER TABLE survey_responses_archive ALTER COLUMN "updatedAt" DROP NOT NULL;
+
+ALTER TABLE survey_responses_archive ALTER COLUMN response SET DEFAULT '[]'::jsonb;
+
+ALTER TABLE solicitations ADD CONSTRAINT "uq_solicitations_solNum" UNIQUE ("solNum");
+
+ALTER TABLE solicitations ALTER COLUMN history SET DEFAULT '[]'::jsonb;
+
+ALTER TABLE solicitations ALTER COLUMN action SET DEFAULT '[]'::jsonb;
+
+ALTER TABLE solicitations ALTER COLUMN predictions SET DEFAULT '{"value": "red", "history": []}'::jsonb;
+
+ALTER TABLE solicitations ALTER COLUMN compliant SET DEFAULT 0;
+
+ALTER TABLE solicitations ALTER COLUMN active SET DEFAULT true;
+
+ALTER TABLE solicitations ALTER COLUMN na_flag SET DEFAULT false;
+
+ALTER TABLE solicitations ALTER COLUMN "updateAt" DROP NOT NULL;
+
+ALTER TABLE attachment ALTER COLUMN "createdAt" SET NOT NULL;
+
+ALTER TABLE attachment ALTER COLUMN "createdAt" SET DEFAULT now();
+
+ALTER TABLE notice ALTER COLUMN "createdAt" SET NOT NULL;
+
+ALTER TABLE notice ALTER COLUMN "createdAt" SET DEFAULT now();
+
+
+COMMIT;
+

src/fbo_scraper/db/db.py

@@ -215,6 +215,15 @@ class Solicitation(Base):
         "Attachment", back_populates="solicitation", cascade="all, delete-orphan"
     )
 
+class ARTLanguage(Base):
+    __tablename__ = "art_language"
+    id = Column(Integer, primary_key=True)
+    solicitation_id = Column(Integer, ForeignKey("solicitations.id"))
+    language = Column(JSONB, nullable=False)
+    createdAt = Column(DateTime, nullable=False, default=func.now())
+    updatedAt = Column(DateTime, onupdate=func.now())
+
+    solicitation = relationship("Solicitation", back_populates="art_language")
 
 class SurveyResponse(Base):
     __tablename__ = "survey_responses"

src/fbo_scraper/util/ebuy_csv.py

@@ -411,7 +411,7 @@ def ebuy_process(options):
     logger.debug(options)
 
     rfq_data = parse_csv(options.file_path)
-    logger.debug("After Parse: ", rfq_data[0])
+    #logger.debug("After Parse: ", rfq_data[0])
 
     if not check_for_ebuy_headers(rfq_data):
         logger.error("Missing necessary columns in the eBuy CSV")
@@ -424,18 +424,18 @@ def ebuy_process(options):
     predict = Predict(best_model_path=options.model_path)
 
     rfq_data = filter_out_no_attachments(rfq_data)
-    logger.debug("After Filter: ", rfq_data[0])
+    #logger.debug("After Filter: ", rfq_data[0])
 
     rfq_data = filter_out_no_naics(rfq_data)
-    logger.debug("After NAICS Filter: ", rfq_data[0])
+    #logger.debug("After NAICS Filter: ", rfq_data[0])
 
     rfq_data = rfq_relabeling(rfq_data)
-    logger.debug("After Labeling: ", rfq_data[0])
+    #logger.debug("After Labeling: ", rfq_data[0])
 
     rfq_data = grab_attachment_texts(rfq_data)
 
     predicted_data = predict.insert_predictions(rfq_data)
-    logger.debug(predicted_data[0])
+    #logger.debug(predicted_data[0])
 
     with dal.Session.begin() as session:
         if predicted_data: