Fixes a few things in the events system (#157)

README.md

@@ -14,6 +14,7 @@ This application provides a central system to handle ecommerce activities across
   - [Committing \& Formatting](#committing--formatting)
   - [Optional Setup](#optional-setup)
     - [Interstitial Debug Mode](#interstitial-debug-mode)
+    - [Webhook Retry](#webhook-retry)
     - [Running the app in a notebook](#running-the-app-in-a-notebook)
 
 ## Initial Setup
@@ -189,6 +190,15 @@ Described below are some setup steps that are not strictly necessary for running
 
 You can set `MITOL_UE_PAYMENT_INTERSTITIAL_DEBUG` to control whether or not the checkout interstitial page displays additional data and waits to submit or not. By default, this tracks the `DEBUG` setting (so it should be off in production, and on in local testing).
 
+### Webhook Retry
+
+The events system will attempt to ping integrated systems via a webhook when orders hit certain states (such as completed or refunded). You can control how this works with these settings:
+
+- `MITOL_UE_WEBHOOK_RETRY_MAX` - Max number of attempts to make to hit a webhook. Defaults to 4.
+- `MITOL_UE_WEBHOOK_RETRY_COOLDOWN` - How long to wait between retrying, in seconds. Defaults to 60 seconds.
+
+The retry happens if the request times out, returns an HTTP error, or returns a connection error. If the webhook isn't configured with a URL, if it returns non-JSON data or a redirect loop, or some other error happens, the system _will not_ retry the webhook and an error message will be emitted to that effect. Similarly, if it falls out the end of the available retries it will also emit an error message and stop.
+
 ### Running the app in a notebook
 
 This repo includes a config for running a [Jupyter notebook](https://jupyter.org/) in a Docker container. This enables you to do in a Jupyter notebook anything you might otherwise do in a Django shell. To get started:

cart/views.py

@@ -26,11 +26,9 @@
     USER_MSG_TYPE_PAYMENT_ERROR,
     USER_MSG_TYPE_PAYMENT_ERROR_UNKNOWN,
 )
-from unified_ecommerce.plugin_manager import get_plugin_manager
 from unified_ecommerce.utils import redirect_with_user_message
 
 log = logging.getLogger(__name__)
-pm = get_plugin_manager()
 
 
 class CartView(LoginRequiredMixin, TemplateView):
@@ -150,9 +148,10 @@ def post(self, request):
 
             if order.state == Order.STATE.PENDING:
                 processed_order_state = api.process_cybersource_payment_response(
-                    request, order
+                    request,
+                    order,
+                    POST_SALE_SOURCE_REDIRECT,
                 )
-                pm.hook.post_sale(order_id=order.id, source=POST_SALE_SOURCE_REDIRECT)
 
                 return self.post_checkout_redirect(processed_order_state, request)
             else:

payments/api.py

@@ -18,11 +18,14 @@
     Order,
     PendingOrder,
 )
+from payments.tasks import send_post_sale_webhook
 from system_meta.models import IntegratedSystem
 from unified_ecommerce.constants import (
     CYBERSOURCE_ACCEPT_CODES,
     CYBERSOURCE_ERROR_CODES,
     CYBERSOURCE_REASON_CODE_SUCCESS,
+    POST_SALE_SOURCE_BACKOFFICE,
+    POST_SALE_SOURCE_REDIRECT,
     REFUND_SUCCESS_STATES,
     USER_MSG_TYPE_PAYMENT_ACCEPTED_NOVALUE,
     ZERO_PAYMENT_DATA,
@@ -101,9 +104,11 @@ def generate_checkout_payload(request):
     )
 
 
-def fulfill_completed_order(order, payment_data, basket=None):
+def fulfill_completed_order(
+    order, payment_data, basket=None, source=POST_SALE_SOURCE_BACKOFFICE
+):
     """Fulfill the order."""
-    order.fulfill(payment_data)
+    order.fulfill(payment_data, source)
     order.save()
 
     if basket and basket.compare_to_order(order):
@@ -125,7 +130,9 @@ def get_order_from_cybersource_payment_response(request):
     return order
 
 
-def process_cybersource_payment_response(request, order):
+def process_cybersource_payment_response(
+    request, order, source=POST_SALE_SOURCE_REDIRECT
+):
     """
     Update the order and basket based on the payment request from Cybersource.
     Returns the order state after applying update operations corresponding to
@@ -205,7 +212,7 @@ def process_cybersource_payment_response(request, order):
         try:
             msg = f"Transaction accepted!: {processor_response.message}"
             log.debug(msg)
-            fulfill_completed_order(order, request.POST, basket)
+            fulfill_completed_order(order, request.POST, basket, source)
         except ValidationError:
             msg = (
                 "Missing transaction id from transaction response: "
@@ -403,3 +410,32 @@ def check_and_process_pending_orders_for_resolution(refnos=None):
                 error_count += 1
 
     return (fulfilled_count, cancel_count, error_count)
+
+
+def process_post_sale_webhooks(order_id, source):
+    """
+    Send data to the webhooks for post-sale events.
+
+    If the system in question doesn't have a webhook URL, we will skip it.
+    """
+
+    log.info("Queueing webhook endpoints for order %s with source %s", order_id, source)
+
+    order = Order.objects.prefetch_related("lines", "lines__product_version").get(
+        pk=order_id
+    )
+
+    systems = [
+        product.system
+        for product in [
+            line.product_version._object_version.object  # noqa: SLF001
+            for line in order.lines.all()
+        ]
+    ]
+
+    for system in systems:
+        if not system.webhook_url:
+            log.warning("No webhook URL specified for system %s", system.slug)
+            continue
+
+        send_post_sale_webhook.delay(system.id, order.id, source)

payments/api_test.py

@@ -10,15 +10,19 @@
 from django.urls import reverse
 from factory import Faker, fuzzy
 from mitol.payment_gateway.api import PaymentGateway, ProcessorResponse
+from reversion.models import Version
 
 from payments.api import (
     check_and_process_pending_orders_for_resolution,
     generate_checkout_payload,
     process_cybersource_payment_response,
+    process_post_sale_webhooks,
     refund_order,
 )
+from payments.constants import PAYMENT_HOOK_ACTION_POST_SALE
 from payments.exceptions import PaymentGatewayError, PaypalRefundError
 from payments.factories import (
+    LineFactory,
     OrderFactory,
     TransactionFactory,
 )
@@ -29,8 +33,12 @@
     Order,
     Transaction,
 )
+from payments.serializers.v0 import WebhookBase, WebhookBaseSerializer, WebhookOrder
 from system_meta.factories import ProductFactory
+from system_meta.models import IntegratedSystem
 from unified_ecommerce.constants import (
+    POST_SALE_SOURCE_BACKOFFICE,
+    POST_SALE_SOURCE_REDIRECT,
     TRANSACTION_TYPE_PAYMENT,
     TRANSACTION_TYPE_REFUND,
 )
@@ -90,6 +98,21 @@ def fulfilled_paypal_transaction(fulfilled_order):
     )
 
 
+@pytest.fixture()
+def fulfilled_complete_order():
+    """Create a fulfilled order with line items."""
+
+    order = OrderFactory.create(state=Order.STATE.FULFILLED)
+
+    with reversion.create_revision():
+        product = ProductFactory.create()
+
+    product_version = Version.objects.get_for_object(product).first()
+    LineFactory.create(order=order, product_version=product_version)
+
+    return order
+
+
 @pytest.fixture()
 def products():
     """Create products"""
@@ -385,6 +408,7 @@ def test_process_cybersource_payment_response(rf, mocker, user, products):
     Test that ensures the response from Cybersource for an ACCEPTed payment
     updates the orders state
     """
+    mocker.patch("requests.post")
     mocker.patch(
         "mitol.payment_gateway.api.PaymentGateway.validate_processor_response",
         return_value=True,
@@ -551,3 +575,82 @@ def test_check_and_process_pending_orders_for_resolution(mocker, test_type):
         order.refresh_from_db()
         assert order.state == Order.STATE.FULFILLED
         assert (fulfilled, cancelled, errored) == (1, 0, 0)
+
+
+@pytest.mark.parametrize(
+    "source", [POST_SALE_SOURCE_BACKOFFICE, POST_SALE_SOURCE_REDIRECT]
+)
+def test_integrated_system_webhook(mocker, fulfilled_complete_order, source):
+    """Test fire the webhook."""
+
+    mocked_request = mocker.patch("requests.post")
+    system_id = fulfilled_complete_order.lines.first().product_version.field_dict[
+        "system_id"
+    ]
+    system = IntegratedSystem.objects.get(pk=system_id)
+
+    order_info = WebhookOrder(
+        order=fulfilled_complete_order,
+        lines=fulfilled_complete_order.lines.all(),
+    )
+
+    webhook_data = WebhookBase(
+        type=PAYMENT_HOOK_ACTION_POST_SALE,
+        system_key=system.api_key,
+        user=fulfilled_complete_order.purchaser,
+        data=order_info,
+    )
+
+    serialized_webhook_data = WebhookBaseSerializer(webhook_data)
+
+    process_post_sale_webhooks(fulfilled_complete_order.id, source)
+
+    mocked_request.assert_called_with(
+        system.webhook_url, json=serialized_webhook_data.data, timeout=30
+    )
+
+
+@pytest.mark.parametrize(
+    "source", [POST_SALE_SOURCE_BACKOFFICE, POST_SALE_SOURCE_REDIRECT]
+)
+def test_integrated_system_webhook_multisystem(
+    mocker, fulfilled_complete_order, source
+):
+    """Test fire the webhook with an order with lines from >1 system."""
+
+    with reversion.create_revision():
+        product = ProductFactory.create()
+
+    product_version = Version.objects.get_for_object(product).first()
+    LineFactory.create(order=fulfilled_complete_order, product_version=product_version)
+
+    mocked_request = mocker.patch("requests.post")
+
+    serialized_calls = []
+
+    for system in IntegratedSystem.objects.all():
+        order_info = WebhookOrder(
+            order=fulfilled_complete_order,
+            lines=[
+                line
+                for line in fulfilled_complete_order.lines.all()
+                if line.product.system.slug == system.slug
+            ],
+        )
+
+        webhook_data = WebhookBase(
+            type=PAYMENT_HOOK_ACTION_POST_SALE,
+            system_key=system.api_key,
+            user=fulfilled_complete_order.purchaser,
+            data=order_info,
+        )
+
+        serialized_order = WebhookBaseSerializer(webhook_data).data
+        serialized_calls.append(
+            mocker.call(system.webhook_url, json=serialized_order, timeout=30)
+        )
+
+    process_post_sale_webhooks(fulfilled_complete_order.id, source)
+
+    assert mocked_request.call_count == 2
+    mocked_request.assert_has_calls(serialized_calls, any_order=True)

payments/hooks/post_sale.py

@@ -3,11 +3,6 @@
 import logging
 
 import pluggy
-import requests
-
-from payments.constants import PAYMENT_HOOK_ACTION_POST_SALE
-from payments.models import Order
-from payments.serializers.v0 import WebhookBase, WebhookBaseSerializer, WebhookOrder
 
 hookimpl = pluggy.HookimplMarker("unified_ecommerce")
 
@@ -27,64 +22,9 @@ def post_sale(self, order_id, source):
 class IntegratedSystemWebhooks:
     """Figures out what webhook endpoints to call, and calls them."""
 
-    def post_sale_impl(self, order_id, source):
-        """Call the webhook endpoints for the order."""
-
-        log = logging.getLogger(__name__)
-
-        log.info(
-            "Calling webhook endpoints for order %s with source %s", order_id, source
-        )
-
-        order = Order.objects.prefetch_related("lines", "lines__product_version").get(
-            pk=order_id
-        )
-
-        systems = [
-            product.system
-            for product in [
-                line.product_version._object_version.object  # noqa: SLF001
-                for line in order.lines.all()
-            ]
-        ]
-
-        for system in systems:
-            system_webhook_url = system.webhook_url
-            system_slug = system.slug
-            if system_webhook_url:
-                log.info(
-                    ("Calling webhook endpoint %s for order %s with source %s"),
-                    system_webhook_url,
-                    order_id,
-                    source,
-                )
-
-                order_info = WebhookOrder(
-                    order=order,
-                    lines=[
-                        line
-                        for line in order.lines.all()
-                        if line.product.system.slug == system_slug
-                    ],
-                )
-
-                webhook_data = WebhookBase(
-                    type=PAYMENT_HOOK_ACTION_POST_SALE,
-                    system_key=system.api_key,
-                    user=order.purchaser,
-                    data=order_info,
-                )
-
-                serializer = WebhookBaseSerializer(webhook_data)
-
-                requests.post(
-                    system_webhook_url,
-                    json=serializer.data,
-                    timeout=30,
-                )
-
     @hookimpl
     def post_sale(self, order_id, source):
         """Call the implementation of this, so we can test it more easily."""
+        from payments.api import process_post_sale_webhooks
 
-        self.post_sale_impl(order_id, source)
+        process_post_sale_webhooks(order_id, source)