Merge branch 'main' into mikehgrantsgov/3536-add-saved-notifications-to-job

Author: mikehgrantsgov

.github/workflows/ci-frontend-e2e.yml

@@ -46,6 +46,7 @@ jobs:
       - name: Start API Server for e2e tests
         run: |
           cd ../api
+          echo "ENABLE_OPPORTUNITY_ATTACHMENT_PIPELINE=false" >> override.env
           make init db-seed-local populate-search-opportunities start &
           cd ../frontend
           # Ensure the API wait script is executable

api/openapi.generated.yml

@@ -1327,6 +1327,18 @@ components:
           - string
           - 'null'
           format: date
+        start_date_relative:
+          type:
+          - integer
+          - 'null'
+          minimum: -1000000
+          maximum: 1000000
+        end_date_relative:
+          type:
+          - integer
+          - 'null'
+          minimum: -1000000
+          maximum: 1000000
     CloseDateFilterV1:
       type: object
       properties:
@@ -1340,6 +1352,18 @@ components:
           - string
           - 'null'
           format: date
+        start_date_relative:
+          type:
+          - integer
+          - 'null'
+          minimum: -1000000
+          maximum: 1000000
+        end_date_relative:
+          type:
+          - integer
+          - 'null'
+          minimum: -1000000
+          maximum: 1000000
     OpportunitySearchFilterV1:
       type: object
       properties:

api/src/adapters/search/opensearch_query_builder.py

@@ -194,8 +194,19 @@ def filter_int_range(
         self.filters.append({"range": {field: range_filter}})
         return self
 
+    def adjust_date_format(self, in_date: datetime.date | int | None) -> str | None:
+        if in_date is None:
+            return None
+        if isinstance(in_date, int):
+            return f"now{in_date:+}d"
+
+        return in_date.isoformat()
+
     def filter_date_range(
-        self, field: str, start_date: datetime.date | None, end_date: datetime.date | None
+        self,
+        field: str,
+        start_date: datetime.date | int | None,
+        end_date: datetime.date | int | None,
     ) -> typing.Self:
         """
         For a given field, filter results to a range of dates.
@@ -207,13 +218,16 @@ def filter_date_range(
         a binary filter on the overall results.
         """
         if start_date is None and end_date is None:
-            raise ValueError("Cannot use date range filter if both start and end are None")
+            raise ValueError("Cannot use date range filter if both start and end dates are None")
+
+        start_date_str = self.adjust_date_format(start_date)
+        end_date_str = self.adjust_date_format(end_date)
 
         range_filter = {}
-        if start_date is not None:
-            range_filter["gte"] = start_date.isoformat()
-        if end_date is not None:
-            range_filter["lte"] = end_date.isoformat()
+        if start_date_str is not None:
+            range_filter["gte"] = start_date_str
+        if end_date_str is not None:
+            range_filter["lte"] = end_date_str
 
         self.filters.append({"range": {field: range_filter}})
         return self

api/src/api/schemas/search_schema.py

@@ -293,6 +293,14 @@ class DateSearchSchemaBuilder(BaseSearchSchemaBuilder):
     def with_date_range(self) -> "DateSearchSchemaBuilder":
         self.schema_fields["start_date"] = fields.Date(allow_none=True)
         self.schema_fields["end_date"] = fields.Date(allow_none=True)
+
+        self.schema_fields["start_date_relative"] = fields.Integer(
+            allow_none=True, validate=[validators.Range(min=-1000000, max=1000000)]
+        )
+        self.schema_fields["end_date_relative"] = fields.Integer(
+            allow_none=True, validate=[validators.Range(min=-1000000, max=1000000)]
+        )
+
         self._with_date_range_validator()
 
         return self
@@ -302,16 +310,38 @@ def _with_date_range_validator(self) -> "DateSearchSchemaBuilder":
         # rules that go across fields in the validation
         @validates_schema
         def validate_date_range(_: Any, data: dict, **kwargs: Any) -> None:
+
             start_date = data.get("start_date", None)
             end_date = data.get("end_date", None)
 
-            # Error if start and end date are None (either explicitly set, or because they are missing)
-            if start_date is None and end_date is None:
+            start_date_relative = data.get("start_date_relative", None)
+            end_date_relative = data.get("end_date_relative", None)
+
+            # Error if both relative date and absolute date provided for either start or end date
+            if ("start_date" in data and "start_date_relative" in data) or (
+                "end_date" in data and "end_date_relative" in data
+            ):
+                raise ValidationError(
+                    [
+                        MarshmallowErrorContainer(
+                            ValidationErrorType.INVALID,
+                            "Cannot have both absolute and relative start/end date.",
+                        )
+                    ]
+                )
+
+            # Error if both start and end date for either relative or absolute date are None (either explicitly set, or because they are missing)
+            if (
+                start_date is None
+                and end_date is None
+                and start_date_relative is None
+                and end_date_relative is None
+            ):
                 raise ValidationError(
                     [
                         MarshmallowErrorContainer(
                             ValidationErrorType.REQUIRED,
-                            "At least one of start_date or end_date must be provided.",
+                            "At least one of start_date/start_date_relative or end_date/end_date_relative must be provided.",
                         )
                     ]
                 )

api/src/data_migration/transformation/subtask/transform_agency.py

@@ -73,6 +73,8 @@
     # These fields were only found in the test environment
     "ASSISTCompatible",
     "SAMValidation",
+    # This was added in Jan 2025 in Grants.gov, we aren't using it yet
+    "AllowSubmitWithExpSAM",
 }
 
 REQUIRED_FIELDS = {

api/src/db/migrations/versions/2025_01_28_add_searched_opportunity_ids_and_last_.py

@@ -0,0 +1,44 @@
+"""add searched_opportunity_ids_and_last_notified_at to user_saved_opportunity
+
+Revision ID: 9e7fc937646a
+Revises: dc04ce955a9a
+Create Date: 2025-01-28 19:06:19.397240
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "9e7fc937646a"
+down_revision = "dc04ce955a9a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "user_saved_search",
+        sa.Column(
+            "last_notified_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        schema="api",
+    )
+    op.add_column(
+        "user_saved_search",
+        sa.Column("searched_opportunity_ids", postgresql.ARRAY(sa.BigInteger()), nullable=False),
+        schema="api",
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("user_saved_search", "searched_opportunity_ids", schema="api")
+    op.drop_column("user_saved_search", "last_notified_at", schema="api")
+    # ### end Alembic commands ###

api/src/db/models/user_models.py

@@ -2,14 +2,16 @@
 from datetime import datetime
 
 from sqlalchemy import BigInteger, ForeignKey
-from sqlalchemy.dialects.postgresql import JSONB, UUID
+from sqlalchemy.dialects.postgresql import ARRAY, JSONB, UUID
 from sqlalchemy.orm import Mapped, mapped_column, relationship
+from sqlalchemy.sql.functions import now as sqlnow
 
 from src.adapters.db.type_decorators.postgres_type_decorators import LookupColumn
 from src.constants.lookup_constants import ExternalUserType
 from src.db.models.base import ApiSchemaTable, TimestampMixin
 from src.db.models.lookup_models import LkExternalUserType
 from src.db.models.opportunity_models import Opportunity
+from src.util import datetime_util
 
 
 class User(ApiSchemaTable, TimestampMixin):
@@ -106,6 +108,13 @@ class UserSavedSearch(ApiSchemaTable, TimestampMixin):
 
     name: Mapped[str]
 
+    last_notified_at: Mapped[datetime] = mapped_column(
+        nullable=False,
+        default=datetime_util.utcnow,
+        server_default=sqlnow(),
+    )
+    searched_opportunity_ids: Mapped[list[int]] = mapped_column(ARRAY(BigInteger))
+
 
 class UserNotificationLog(ApiSchemaTable, TimestampMixin):
     __tablename__ = "user_notification_log"

api/src/search/search_models.py

@@ -19,3 +19,5 @@ class IntSearchFilter(BaseModel):
 class DateSearchFilter(BaseModel):
     start_date: date | None = None
     end_date: date | None = None
+    start_date_relative: int | None = None
+    end_date_relative: int | None = None

api/src/services/opportunities_v1/search_opportunities.py

@@ -127,7 +127,21 @@ def _add_search_filters(
             builder.filter_int_range(field_name, field_filters.min, field_filters.max)
 
         elif isinstance(field_filters, DateSearchFilter):
-            builder.filter_date_range(field_name, field_filters.start_date, field_filters.end_date)
+            start_date = (
+                field_filters.start_date
+                if field_filters.start_date
+                else field_filters.start_date_relative
+            )
+            end_date = (
+                field_filters.end_date
+                if field_filters.end_date
+                else field_filters.end_date_relative
+            )
+            builder.filter_date_range(
+                field_name,
+                start_date,
+                end_date,
+            )
 
 
 def _add_aggregations(builder: search.SearchQueryBuilder) -> None:

api/src/services/users/create_saved_search.py

@@ -9,7 +9,10 @@
 
 def create_saved_search(db_session: db.Session, user_id: UUID, json_data: dict) -> UserSavedSearch:
     saved_search = UserSavedSearch(
-        user_id=user_id, name=json_data["name"], search_query=json_data["search_query"]
+        user_id=user_id,
+        name=json_data["name"],
+        search_query=json_data["search_query"],
+        searched_opportunity_ids=[],
     )
 
     db_session.add(saved_search)

api/tests/src/adapters/search/test_opensearch_query_builder.py

@@ -390,19 +390,32 @@ def test_query_builder_filter_terms(
         "start_date,end_date,expected_results",
         [
             # Date range that will include all results
+            # Absolute
             (date(1900, 1, 1), date(2050, 1, 1), FULL_DATA),
+            # Relative
+            (-45656, 9131, FULL_DATA),
             # Start only date range that will get all results
+            # Absolute
             (date(1950, 1, 1), None, FULL_DATA),
+            # Relative
+            (-45656, None, FULL_DATA),
             # End only date range that will get all results
+            # Absolute
             (None, date(2025, 1, 1), FULL_DATA),
+            # Relative
+            (None, 9131, FULL_DATA),
             # Range that filters to just oldest
             (
                 date(1950, 1, 1),
                 date(1960, 1, 1),
                 [FELLOWSHIP_OF_THE_RING, TWO_TOWERS, RETURN_OF_THE_KING],
             ),
             # Unbounded range for oldest few
-            (None, date(1990, 1, 1), [FELLOWSHIP_OF_THE_RING, TWO_TOWERS, RETURN_OF_THE_KING]),
+            (
+                None,
+                date(1990, 1, 1),
+                [FELLOWSHIP_OF_THE_RING, TWO_TOWERS, RETURN_OF_THE_KING],
+            ),
             # Unbounded range for newest few
             (date(2011, 8, 1), None, [WORDS_OF_RADIANCE, OATHBRINGER, RHYTHM_OF_WAR]),
             # Selecting a few in the middle
@@ -412,13 +425,24 @@ def test_query_builder_filter_terms(
                 [WAY_OF_KINGS, FEAST_FOR_CROWS, DANCE_WITH_DRAGONS],
             ),
             # Exact date
+            # Absolute
             (date(1954, 7, 29), date(1954, 7, 29), [FELLOWSHIP_OF_THE_RING]),
+            # Relative
+            (-25747, -25747, []),
             # None fetched in range
+            # Absolute
             (date(1981, 1, 1), date(1989, 1, 1), []),
+            # Relative
+            (-16093, -13171, []),
         ],
     )
     def test_query_builder_filter_date_range(
-        self, search_client, search_index, start_date, end_date, expected_results
+        self,
+        search_client,
+        search_index,
+        start_date,
+        end_date,
+        expected_results,
     ):
         builder = (
             SearchQueryBuilder()
@@ -428,9 +452,13 @@ def test_query_builder_filter_date_range(
 
         expected_ranges = {}
         if start_date is not None:
-            expected_ranges["gte"] = start_date.isoformat()
+            expected_ranges["gte"] = (
+                f"now{start_date:+}d" if isinstance(start_date, int) else start_date.isoformat()
+            )
         if end_date is not None:
-            expected_ranges["lte"] = end_date.isoformat()
+            expected_ranges["lte"] = (
+                f"now{end_date:+}d" if isinstance(end_date, int) else end_date.isoformat()
+            )
 
         expected_query = {
             "size": 25,
@@ -509,7 +537,7 @@ def test_filter_int_range_both_none(self):
         with pytest.raises(ValueError, match="Cannot use int range filter"):
             SearchQueryBuilder().filter_int_range("test_field", None, None)
 
-    def test_filter_date_range_both_none(self):
+    def test_filter_date_range_all_none(self):
         with pytest.raises(ValueError, match="Cannot use date range filter"):
             SearchQueryBuilder().filter_date_range("test_field", None, None)