WIP

.dev/mailpit/Dockerfile

@@ -0,0 +1,18 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM axllent/mailpit:v1.28.0
+
+EXPOSE 1025
+EXPOSE 8025

.dev/mailpit/manifests/pod.yaml

@@ -0,0 +1,41 @@
+# Copyright 2025 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: mailpit
+  labels:
+    app.kubernetes.io/name: mailpit
+spec:
+  containers:
+    - name: mailpit
+      image: mailpit
+      imagePullPolicy: Never # Need this for pushing directly into minikube
+      ports:
+        - containerPort: 1025
+          name: smtp-port
+        - containerPort: 8025
+          name: web-ui-port
+      readinessProbe:
+        tcpSocket:
+          port: 1025
+        initialDelaySeconds: 10
+      resources:
+        limits:
+          cpu: 250m
+          memory: 128Mi
+        requests:
+          cpu: 100m
+          memory: 64Mi

.dev/mailpit/manifests/service.yaml

@@ -0,0 +1,30 @@
+# Copyright 2025 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: mailpit
+spec:
+  selector:
+    app.kubernetes.io/name: mailpit
+  ports:
+    - name: smtp
+      protocol: TCP
+      port: 1025
+      targetPort: smtp-port
+    - name: web-ui
+      protocol: TCP
+      port: 8025
+      targetPort: web-ui-port

.dev/mailpit/skaffold.yaml

@@ -0,0 +1,31 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: skaffold/v4beta9
+kind: Config
+metadata:
+  name: mailpit-config
+profiles:
+  - name: local
+    build:
+      artifacts:
+        - image: mailpit
+          context: .
+      local:
+        useBuildkit: true
+    manifests:
+      rawYaml:
+        - manifests/*
+    deploy:
+      kubectl: {}

.dev/pubsub/run.sh

@@ -32,9 +32,11 @@ create_topic() {
 }
 
 # Function to create a Subscription (Pull). gcloud does not support subscription creation in the emulator, so we use curl.
+# Usage: create_subscription <topic_name> <sub_name> [dlq_topic_name]
 create_subscription() {
     local topic_name=$1
     local sub_name=$2
+    local dlq_topic_name=$3
 
     if [[ -z "$topic_name" || -z "$sub_name" ]]; then
         echo "Error: Topic name and Subscription name required."
@@ -46,25 +48,60 @@ create_subscription() {
     # The emulator requires the full path to the topic in the JSON body
     local topic_path="projects/${PROJECT_ID}/topics/${topic_name}"
 
+    # Base JSON payload
+    local json_payload="{\"topic\": \"$topic_path\""
+
+    # If a DLQ topic is provided, add the deadLetterPolicy
+    if [[ -n "$dlq_topic_name" ]]; then
+        local dlq_topic_path="projects/${PROJECT_ID}/topics/${dlq_topic_name}"
+        # MaxDeliveryAttempts is set to 5 as a standard default
+        json_payload="${json_payload}, \"deadLetterPolicy\": {\"deadLetterTopic\": \"${dlq_topic_path}\", \"maxDeliveryAttempts\": 5}"
+        echo "  -> With Dead Letter Queue: ${dlq_topic_name}"
+    fi
+
+    # Close the JSON object
+    json_payload="${json_payload}}"
+
     curl -s -X PUT "http://0.0.0.0:${PORT}/v1/projects/${PROJECT_ID}/subscriptions/${sub_name}" \
         -H "Content-Type: application/json" \
-        -d "{\"topic\": \"$topic_path\"}"
+        -d "$json_payload"
 
     echo -e "\nSubscription ${sub_name} for topic: ${topic_name} created."
 }
 
 gcloud beta emulators pubsub start --project="$PROJECT_ID" --host-port="0.0.0.0:$PORT" &
-while ! curl -s -o /dev/null "localhost:$PORT"; do
+while ! curl -s -f "http://0.0.0.0:${PORT}/v1/projects/${PROJECT_ID}/topics"; do
   sleep 1 # Wait 1 second before checking again
   echo "waiting until pubsub emulator responds before finishing setup"
 done
 
+# --- 1. Dead Letter Queues (Create these first so main subs can reference them) ---
+
+# Ingestion DLQ
+create_topic "ingestion-jobs-dead-letter-topic-id"
+create_subscription "ingestion-jobs-dead-letter-topic-id" "ingestion-jobs-dead-letter-sub-id"
+
+# Notification/Fan-out DLQ
+create_topic "notification-events-dead-letter-topic-id"
+create_subscription "notification-events-dead-letter-topic-id" "notification-events-dead-letter-sub-id"
+
+# Delivery DLQ
+create_topic "delivery-dead-letter-topic-id"
+create_subscription "delivery-dead-letter-topic-id" "delivery-dead-letter-sub-id"
+
+
+# --- 2. Main Topics and Subscriptions ---
+create_topic "batch-updates-topic-id"
+create_subscription "batch-updates-topic-id" "batch-updates-sub-id" "ingestion-jobs-dead-letter-topic-id"
+
 create_topic "ingestion-jobs-topic-id"
-create_subscription "ingestion-jobs-topic-id" "ingestion-jobs-sub-id"
+create_subscription "ingestion-jobs-topic-id" "ingestion-jobs-sub-id" "ingestion-jobs-dead-letter-topic-id"
+
 create_topic "notification-events-topic-id"
-create_subscription "notification-events-topic-id" "notification-events-sub-id"
+create_subscription "notification-events-topic-id" "notification-events-sub-id" "notification-events-dead-letter-topic-id"
+
 create_topic "chime-delivery-topic-id"
-create_subscription "chime-delivery-topic-id" "chime-delivery-sub-id"
+create_subscription "chime-delivery-topic-id" "chime-delivery-sub-id" "delivery-dead-letter-topic-id"
 
 echo "Pub/Sub setup for webstatus.dev finished"
 

.prettierignore

@@ -7,6 +7,7 @@ jsonschema/mdn_browser-compat-data
 jsonschema/web-platform-dx_web-features
 jsonschema/web-platform-dx_web-features-mappings
 docs/schema
+workers/email/pkg/digest/testdata/digest.golden.html
 workflows/steps/services/bcd_consumer/pkg/data/testdata/data.json
 workflows/steps/services/web_feature_consumer/pkg/data/testdata/v3.data.json
 workflows/steps/services/developer_signals_consumer/pkg/data/testdata/web-features-signals.json

DEVELOPMENT.md

@@ -93,7 +93,7 @@ The above skaffold command deploys multiple resources:
 | valkey    | Valkey                         | N/A                                                        | valkey:6379                                                         |
 | auth      | Auth Emulator                  | http://localhost:9099<br />http://localhost:9100/auth (ui) | http://auth:9099<br />http://auth:9100/auth (ui)                    |
 | wiremock  | Wiremock                       | http://localhost:8087                                      | http://api-github-mock.default.svc.cluster.local:8080 (GitHub Mock) |
-| pubsub    | Pub/Sub Emulator               | N/A                                                        | http://pubsub:8086                                                  |
+| pubsub    | Pub/Sub Emulator               | http://localhost:8060                                      | http://pubsub:8060                                                  |
 | gcs       | GCS Emulator                   | N/A                                                        | http://gcs:4443                                                     |
 
 _In the event the servers are not responsive, make a temporary change to a file_

GEMINI.md

@@ -69,6 +69,8 @@ The frontend Single Page Application (SPA).
   - **DON'T** introduce other UI frameworks like React or Vue.
   - **DO** use the Lit Context and service container pattern to access shared services.
   - **DON'T** create new global state management solutions.
+  - **DON'T** render the full page layout (header, sidebar, main container) inside a page component. DO focus on rendering only the specific content for that route. The main `webstatus-app` component provides the overall page shell.
+  - **DON'T** add generic class names (e.g., `.card`, `.container`) to `shared-css.ts`. DO leverage Shadow DOM encapsulation by defining component-specific styles within the component's `static styles` block. Use composition with slots (e.g., a `<webstatus-notification-panel>` component) for reusable layout patterns instead of shared global CSS classes.
   - **DO** add Playwright tests for new user-facing features and unit tests for component logic.
 
 #### Workflows / Data Ingestion Jobs (`workflows/`)
@@ -165,17 +167,17 @@ This section describes the components related to user accounts, authentication,
 
 - **Authentication**: User authentication is handled via Firebase Authentication on the frontend, which integrates with GitHub as an OAuth provider. When a user signs in, the frontend receives a Firebase token and a GitHub token. The GitHub token is sent to the backend during a login sync process.
 - **User Profile Sync**: On login, the backend synchronizes the user's verified GitHub emails with their notification channels in the database. This is an example of a transactional update using the mapper pattern. The flow is as follows:
-  1. A user signs in on the frontend. The frontend sends the user's GitHub token to the backend's `/v1/users/me/ping` endpoint.
-  2. The `httpserver.PingUser` handler receives the request. It uses a `UserGitHubClient` to fetch the user's profile and verified emails from the GitHub API.
-  3. The handler then calls `spanneradapters.Backend.SyncUserProfileInfo` with the user's profile information.
-  4. The `Backend` adapter translates the `backendtypes.UserProfile` to a `gcpspanner.UserProfile` and calls `gcpspanner.Client.SyncUserProfileInfo`.
-  5. `SyncUserProfileInfo` starts a `ReadWriteTransaction`.
-  6. Inside the transaction, it fetches the user's existing `NotificationChannels` and `NotificationChannelStates`.
-  7. It then compares the existing channels with the verified emails from GitHub.
-     - New emails result in new `NotificationChannel` and `NotificationChannelState` records, created using `createWithTransaction`.
-     - Emails that were previously disabled are re-enabled using `updateWithTransaction`.
-     - Channels for emails that are no longer verified are disabled, also using `updateWithTransaction`.
-  8. The entire set of operations is committed atomically. If any step fails, the entire transaction is rolled back.
+  1.  A user signs in on the frontend. The frontend sends the user's GitHub token to the backend's `/v1/users/me/ping` endpoint.
+  2.  The `httpserver.PingUser` handler receives the request. It uses a `UserGitHubClient` to fetch the user's profile and verified emails from the GitHub API.
+  3.  The handler then calls `spanneradapters.Backend.SyncUserProfileInfo` with the user's profile information.
+  4.  The `Backend` adapter translates the `backendtypes.UserProfile` to a `gcpspanner.UserProfile` and calls `gcpspanner.Client.SyncUserProfileInfo`.
+  5.  `SyncUserProfileInfo` starts a `ReadWriteTransaction`.
+  6.  Inside the transaction, it fetches the user's existing `NotificationChannels` and `NotificationChannelStates`.
+  7.  It then compares the existing channels with the verified emails from GitHub.
+      - New emails result in new `NotificationChannel` and `NotificationChannelState` records, created using `createWithTransaction`.
+      - Emails that were previously disabled are re-enabled using `updateWithTransaction`.
+      - Channels for emails that are no longer verified are disabled, also using `updateWithTransaction`.
+  8.  The entire set of operations is committed atomically. If any step fails, the entire transaction is rolled back.
 - **Notification Channels**: The `NotificationChannels` table stores the destinations for notifications (e.g., email addresses). Each channel has a corresponding entry in the `NotificationChannelStates` table, which tracks whether the channel is enabled or disabled.
 - **Saved Search Subscriptions**: Authenticated users can save feature search queries and subscribe to receive notifications when the results of that query change. This is managed through the `UserSavedSearches` and `UserSavedSearchBookmarks` tables.
 
@@ -232,13 +234,15 @@ This practice decouples the core application logic from the exact structure of t
   - **DO** add E2E tests for critical user journeys.
   - **DON'T** write E2E tests for small component-level interactions.
   - **DO** use resilient selectors like `data-testid`.
+  - **DO** move the mouse to a neutral position (e.g., `page.mouse.move(0, 0)`) before taking a screenshot to avoid flaky tests caused by unintended hover effects.
 - **Go Unit & Integration Tests**:
   - **DO** use table-driven unit tests with mocks for dependencies at the adapter layer (`spanneradapters`).
   - **DO** write **integration tests using `testcontainers-go`** for any changes to the `lib/gcpspanner` layer. This is especially critical when implementing or modifying a mapper. These tests must spin up a Spanner emulator and verify the mapper's logic against a real database.
   - When a refactoring changes how errors are handled (e.g., from returning an error to logging a warning and continuing), **DO** update the tests to reflect the new expected behavior. Some test cases might become obsolete and should be removed or updated.
 - **TypeScript Unit Tests**:
   - **TypeScript**: Use `npm run test -w frontend`.
   - **ES Module Testing**: When testing components that use ES module exports directly (e.g., `signInWithPopup` from `firebase/auth`), direct stubbing with Sinon (e.g., `sinon.stub(firebaseAuth, 'signInWithPopup')`) is problematic due to module immutability. Instead, introduce a helper property (e.g., `credentialGetter`) in the component that defaults to the original ES module function but can be overridden with a Sinon stub in tests. This allows for effective mocking of ES module interactions.
+  - **DO** prefer using generic arguments for `querySelector` in tests (e.g., `querySelector<HTMLSlotElement>('slot[name="content"]')`) to improve type safety and avoid unnecessary casting.
 
 ### 5.3. CI/CD (`.github/`)
 

Makefile

@@ -89,6 +89,8 @@ check-local-ports:
 	$(call wait_for_port,9010,spanner)
 	$(call wait_for_port,8086,datastore)
 	$(call wait_for_port,8087,wiremock)
+	$(call wait_for_port,8060,pubsub)
+	$(call wait_for_port,8025,mailpit)
 
 
 port-forward-manual: port-forward-terminate
@@ -98,13 +100,17 @@ port-forward-manual: port-forward-terminate
 	kubectl wait --for=condition=ready pod/datastore
 	kubectl wait --for=condition=ready pod/spanner
 	kubectl wait --for=condition=ready pod/wiremock
+	kubectl wait --for=condition=ready pod/pubsub
+	kubectl wait --for=condition=ready pod/mailpit
 	kubectl port-forward --address 127.0.0.1 pod/frontend 5555:5555 2>&1 >/dev/null &
 	kubectl port-forward --address 127.0.0.1 pod/backend 8080:8080 2>&1 >/dev/null &
 	kubectl port-forward --address 127.0.0.1 pod/auth 9099:9099 2>&1 >/dev/null &
 	kubectl port-forward --address 127.0.0.1 pod/auth 9100:9100 2>&1 >/dev/null &
 	kubectl port-forward --address 127.0.0.1 pod/spanner 9010:9010 2>&1 >/dev/null &
 	kubectl port-forward --address 127.0.0.1 pod/datastore 8086:8086 2>&1 >/dev/null &
 	kubectl port-forward --address 127.0.0.1 pod/wiremock 8087:8080 2>&1 >/dev/null &
+	kubectl port-forward --address 127.0.0.1 pod/pubsub 8060:8060 2>&1 >/dev/null &
+	kubectl port-forward --address 127.0.0.1 pod/mailpit 8025:8025 2>&1 >/dev/null &
 	make check-local-ports
 
 port-forward-terminate:
@@ -115,6 +121,8 @@ port-forward-terminate:
 	fuser -k 9010/tcp || true
 	fuser -k 8086/tcp || true
 	fuser -k 8087/tcp || true
+	fuser -k 8060/tcp || true
+	fuser -k 8025/tcp || true
 
 # Prerequisite target to start minikube if necessary
 minikube-running:
@@ -337,7 +345,8 @@ ADDLICENSE_ARGS := -c "${COPYRIGHT_NAME}" \
 	-ignore 'node_modules/**' \
 	-ignore 'infra/storage/spanner/schema.sql' \
 	-ignore 'antlr/.antlr/**' \
-	-ignore '.devcontainer/cache/**'
+	-ignore '.devcontainer/cache/**' \
+	-ignore 'workers/email/pkg/digest/testdata/digest.golden.html'
 
 license-check: go-install-tools
 	go tool addlicense -check $(ADDLICENSE_ARGS) .

backend/cmd/server/main.go

@@ -28,6 +28,8 @@ import (
 	"github.com/GoogleChrome/webstatus.dev/backend/pkg/httpserver"
 	"github.com/GoogleChrome/webstatus.dev/lib/auth"
 	"github.com/GoogleChrome/webstatus.dev/lib/cachetypes"
+	"github.com/GoogleChrome/webstatus.dev/lib/gcppubsub"
+	"github.com/GoogleChrome/webstatus.dev/lib/gcppubsub/gcppubsubadapters"
 	"github.com/GoogleChrome/webstatus.dev/lib/gcpspanner"
 	"github.com/GoogleChrome/webstatus.dev/lib/gcpspanner/spanneradapters"
 	"github.com/GoogleChrome/webstatus.dev/lib/gds"
@@ -197,11 +199,30 @@ func main() {
 
 	}
 
+	pubsubProjectID := os.Getenv("PUBSUB_PROJECT_ID")
+	if pubsubProjectID == "" {
+		slog.ErrorContext(ctx, "missing pubsub project id")
+		os.Exit(1)
+	}
+
+	ingestionTopicID := os.Getenv("INGESTION_TOPIC_ID")
+	if ingestionTopicID == "" {
+		slog.ErrorContext(ctx, "missing ingestion topic id")
+		os.Exit(1)
+	}
+
+	queueClient, err := gcppubsub.NewClient(ctx, pubsubProjectID)
+	if err != nil {
+		slog.ErrorContext(ctx, "unable to create pub sub client", "error", err)
+		os.Exit(1)
+	}
+
 	srv := httpserver.NewHTTPServer(
 		"8080",
 		baseURL,
 		datastoreadapters.NewBackend(fs),
 		spanneradapters.NewBackend(spannerClient),
+		gcppubsubadapters.NewBackendAdapter(queueClient, ingestionTopicID),
 		cache,
 		routeCacheOptions,
 		func(token string) *httpserver.UserGitHubClient {

backend/go.mod

@@ -23,6 +23,7 @@ require (
 	cloud.google.com/go/logging v1.13.1 // indirect
 	cloud.google.com/go/longrunning v0.7.0 // indirect
 	cloud.google.com/go/monitoring v1.24.3 // indirect
+	cloud.google.com/go/pubsub/v2 v2.0.0 // indirect
 	cloud.google.com/go/secretmanager v1.16.0 // indirect
 	cloud.google.com/go/spanner v1.86.1 // indirect
 	cloud.google.com/go/storage v1.57.2 // indirect