From e17ae3979a12857f8c04d4a6923bd2d2460eb514 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Tue, 20 Feb 2024 10:18:43 -0500 Subject: [PATCH] feat: create aspects-consumer deployment --- .ci/config.yml | 2 + tutoraspects/patches/k8s-deployments | 54 +++++++++++++++++++ .../patches/local-docker-compose-dev-services | 9 ++++ .../patches/local-docker-compose-services | 28 ++++++++++ tutoraspects/patches/openedx-common-settings | 23 ++++++++ ...penedx-dockerfile-post-python-requirements | 2 + tutoraspects/plugin.py | 3 ++ 7 files changed, 121 insertions(+) diff --git a/.ci/config.yml b/.ci/config.yml index f286d81c3..9f34989d1 100644 --- a/.ci/config.yml +++ b/.ci/config.yml @@ -95,3 +95,5 @@ SUPERSET_XAPI_DASHBOARD_SLUG: openedx-xapi ASPECTS_VERSION: 0.91.0 DOCKER_IMAGE_OPENEDX: edunext/openedx-aspects:{{ASPECTS_VERSION}} DOCKER_IMAGE_OPENEDX_DEV: edunext/openedx-aspects-dev:{{ASPECTS_VERSION}} +ASPECTS_ENABLE_EVENT_BUS_CONSUMER: true +ASPECTS_ENABLE_EVENT_BUS_PRODUCER: true diff --git a/tutoraspects/patches/k8s-deployments b/tutoraspects/patches/k8s-deployments index d38c2979f..bbb9f8a5b 100644 --- a/tutoraspects/patches/k8s-deployments +++ b/tutoraspects/patches/k8s-deployments @@ -538,3 +538,57 @@ spec: hostPath: path: /var/log/ {% endif %} + +{% if ASPECTS_ENABLE_EVENT_BUS_CONSUMER %} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: aspects-consumer + labels: + app.kubernetes.io/name: aspects-consumer +spec: + selector: + matchLabels: + app.kubernetes.io/name: aspects-consumer + template: + metadata: + labels: + app.kubernetes.io/name: aspects-consumer + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + containers: + - name: aspects-consumer + image: {{ DOCKER_IMAGE_OPENEDX }} + env: + - name: SERVICE_VARIANT + value: lms + - name: DJANGO_SETTINGS_MODULE + value: lms.envs.tutor.production + volumeMounts: + - mountPath: /openedx/edx-platform/lms/envs/tutor/ + name: settings-lms + - mountPath: /openedx/edx-platform/cms/envs/tutor/ + name: settings-cms + - mountPath: /openedx/config + name: config + - mountPath: /openedx/edx-platform/uwsgi.ini + name: uwsgi-config + subPath: uwsgi.ini + securityContext: + allowPrivilegeEscalation: false + command: | + ./manage.py lms consume_events -t analytics -g event_routing_backends --extra '{"consumer_name": "aspects"}' + volumes: + - name: settings-lms + configMap: + name: openedx-settings-lms + - name: settings-cms + configMap: + name: openedx-settings-cms + - name: config + configMap: + name: openedx-config +{% endif %} diff --git a/tutoraspects/patches/local-docker-compose-dev-services b/tutoraspects/patches/local-docker-compose-dev-services index 7e1ab8e8d..89f83e685 100644 --- a/tutoraspects/patches/local-docker-compose-dev-services +++ b/tutoraspects/patches/local-docker-compose-dev-services @@ -45,3 +45,12 @@ clickhouse: - ../../env/plugins/aspects/apps/clickhouse/dev_config:/etc/clickhouse-server/config.d/ - ../../env/plugins/aspects/apps/clickhouse/users:/etc/clickhouse-server/users.d/ {% endif %} + +{% if ASPECTS_ENABLE_EVENT_BUS_CONSUMER %} +aspects-consumer: + <<: *openedx-service + environment: + DJANGO_SETTINGS_MODULE: lms.envs.tutor.development + command: | + ./manage.py lms consume_events -t analytics -g event_routing_backends --extra '{"consumer_name": "aspects"}' +{% endif %} diff --git a/tutoraspects/patches/local-docker-compose-services b/tutoraspects/patches/local-docker-compose-services index d66840b56..399560de9 100644 --- a/tutoraspects/patches/local-docker-compose-services +++ b/tutoraspects/patches/local-docker-compose-services @@ -95,3 +95,31 @@ vector: - VECTOR_LOG=warn restart: unless-stopped {% endif %} + +{% if ASPECTS_ENABLE_EVENT_BUS_CONSUMER %} +aspects-consumer: + image: {{ DOCKER_IMAGE_OPENEDX }} + stdin_open: true + tty: true + volumes: + # Settings & config + - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro + - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro + - ../apps/openedx/config:/openedx/config:ro + {%- for mount in iter_mounts(MOUNTS, "openedx", "lms") %} + - {{ mount }} + {%- endfor %} + environment: + SERVICE_VARIANT: lms + DJANGO_SETTINGS_MODULE: lms.envs.tutor.production + restart: unless-stopped + command: | + ./manage.py lms consume_events -t analytics -g event_routing_backends --extra '{"consumer_name": "aspects"}' + deploy: + mode: replicated + replicas: {{ ASPECTS_EVENT_BUS_CONSUMER_REPLICAS }} + depends_on: + {% if RUN_MYSQL %}- mysql{% endif %} + {% if RUN_MONGODB %}- mongodb{% endif %} + {% if RUN_REDIS %}- redis{% endif %} +{% endif %} diff --git a/tutoraspects/patches/openedx-common-settings b/tutoraspects/patches/openedx-common-settings index 3cbe0615e..f2ca9fd70 100644 --- a/tutoraspects/patches/openedx-common-settings +++ b/tutoraspects/patches/openedx-common-settings @@ -27,3 +27,26 @@ if not OPEN_EDX_FILTERS_CONFIG.get("org.openedx.learning.instructor.dashboard.re } OPEN_EDX_FILTERS_CONFIG["org.openedx.learning.instructor.dashboard.render.started.v1"]["pipeline"].append("platform_plugin_aspects.extensions.filters.AddSupersetTab") {% endif %} + +{% if ASPECTS_ENABLE_EVENT_BUS_PRODUCER %} +# Update the backends to use the event bus +EVENT_TRACKING_BACKENDS["xapi"]["ENGINE"] = "eventtracking.backends.event_bus.EventBusRoutingBackend" +EVENT_TRACKING_BACKENDS["caliper"]["ENGINE"] = "eventtracking.backends.event_bus.EventBusRoutingBackend" +# Update backend to send events in sync mode +EVENT_TRACKING_BACKENDS["xapi"]["OPTIONS"]["backends"]["xapi"]["ENGINE"] = "event_routing_backends.backends.sync_events_router.SyncEventsRouter" +EVENT_TRACKING_BACKENDS["caliper"]["OPTIONS"]["backends"]["caliper"]["ENGINE"] = "event_routing_backends.backends.sync_events_router.SyncEventsRouter" +# Enable the event bus producer +SEND_TRACKING_EVENT_EMITTED_SIGNAL = True +# Update the event bus producer config +EVENT_BUS_PRODUCER_CONFIG.update( + { + "org.openedx.analytics.tracking.event.emitted.v1": { + "analytics": { + "event_key_field": "tracking_log.name", "enabled": True + } + } + } +) + +INSTALLED_APPS.append("openedx_events") +{% endif %} diff --git a/tutoraspects/patches/openedx-dockerfile-post-python-requirements b/tutoraspects/patches/openedx-dockerfile-post-python-requirements index 8decd527c..8d3455576 100644 --- a/tutoraspects/patches/openedx-dockerfile-post-python-requirements +++ b/tutoraspects/patches/openedx-dockerfile-post-python-requirements @@ -2,3 +2,5 @@ RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ pip install "platform-plugin-aspects==v0.5.0" RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ pip install "edx-event-routing-backends==v8.3.1" +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install "event-tracking>=2.3.1" diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index c94e0c2f5..1f2717385 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -51,6 +51,9 @@ # in the ClickHouse database. Make sure that you understand the legal # consequences of data storage and privacy before turning this on! ("ASPECTS_ENABLE_PII", False), + ("ASPECTS_ENABLE_EVENT_BUS_CONSUMER", False), + ("ASPECTS_ENABLE_EVENT_BUS_PRODUCER", False), + ("ASPECTS_EVENT_BUS_CONSUMER_REPLICAS", 1), # User PII is cached in an in-memory dictionary for this many seconds. ("ASPECTS_PII_CACHE_LIFETIME", 900), # Markdown comprising the Help tab for the Operator and Instructor dashboards.