From b19bd34c007da9ebc15455d40f9f339b930c8de7 Mon Sep 17 00:00:00 2001 From: Matt McMahand Date: Mon, 20 Oct 2025 13:17:57 -0400 Subject: [PATCH 1/2] use received_at for ingesting emails and feed generation --- backend/app/crud/entries.py | 9 ++++++++- backend/app/schemas/entries.py | 1 + backend/app/services/email_processor.py | 6 +++++- backend/app/services/feed_generator.py | 3 +++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/backend/app/crud/entries.py b/backend/app/crud/entries.py index 1cc82bc..643f7c1 100644 --- a/backend/app/crud/entries.py +++ b/backend/app/crud/entries.py @@ -29,9 +29,16 @@ def get_entries_by_newsletter( logger.debug( f"Querying entries for newsletter_id={newsletter_id}, skip={skip}, limit={limit}" ) - query = db.query(Entry).filter(Entry.newsletter_id == newsletter_id).offset(skip) + query = ( + db.query(Entry) + .order_by(Entry.received_at.desc()) + .filter(Entry.newsletter_id == newsletter_id) + .offset(skip) + ) + if limit is not None: query = query.limit(limit) + return query.all() diff --git a/backend/app/schemas/entries.py b/backend/app/schemas/entries.py index ea5afb1..dd93009 100644 --- a/backend/app/schemas/entries.py +++ b/backend/app/schemas/entries.py @@ -9,6 +9,7 @@ class EntryBase(BaseModel): subject: str body: str message_id: str + received_at: datetime.datetime | None = None class EntryCreate(EntryBase): diff --git a/backend/app/services/email_processor.py b/backend/app/services/email_processor.py index 6adaa02..6a9f47f 100644 --- a/backend/app/services/email_processor.py +++ b/backend/app/services/email_processor.py @@ -199,6 +199,8 @@ def _process_single_email( subject = str(make_header(decode_header(msg["Subject"]))) body = _get_email_body(msg) + date_str = msg["Date"] + received_at = email.utils.parsedate_to_datetime(date_str) if date_str else None if newsletter.extract_content: cleaned_data = _extract_and_clean_html(body) @@ -206,7 +208,9 @@ def _process_single_email( # so we only override the body. body = cleaned_data["body"] - entry_schema = EntryCreate(subject=subject, body=body, message_id=message_id) + entry_schema = EntryCreate( + subject=subject, body=body, message_id=message_id, received_at=received_at + ) new_entry = create_entry(db, entry_schema, newsletter.id) if not new_entry: diff --git a/backend/app/services/feed_generator.py b/backend/app/services/feed_generator.py index 879b2c5..de036a9 100644 --- a/backend/app/services/feed_generator.py +++ b/backend/app/services/feed_generator.py @@ -41,11 +41,14 @@ def _add_entries_to_feed( else entry.subject ) fe.content(entry.body, type="html") + if entry.received_at.tzinfo is None: timezone_aware_received_at = entry.received_at.replace(tzinfo=tz.tzutc()) fe.published(timezone_aware_received_at) + fe.updated(timezone_aware_received_at) else: fe.published(entry.received_at) + fe.updated(entry.received_at) def generate_feed(db: Session, feed_identifier: str): From b448aa3f2e9a965888754fcbac1ab86edf7aa5cd Mon Sep 17 00:00:00 2001 From: Matt McMahand Date: Mon, 20 Oct 2025 15:47:23 -0400 Subject: [PATCH 2/2] fix test --- backend/app/tests/test_crud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/tests/test_crud.py b/backend/app/tests/test_crud.py index 7948ae0..72558a7 100644 --- a/backend/app/tests/test_crud.py +++ b/backend/app/tests/test_crud.py @@ -251,7 +251,7 @@ def test_get_entries_by_newsletter(db_session: Session): ) entries = get_entries_by_newsletter(db_session, newsletter.id) assert len(entries) == 2 - assert entries[0].subject == "Entry 1" + assert entries[0].subject == "Entry 2" def test_update_newsletter(db_session: Session):