Merge pull request #927 from openedx/cag/tags

feat: add tag related tables

Author: Cristhian Garcia

tutoraspects/patches/openedx-dev-dockerfile-post-python-requirements

@@ -1,5 +1,5 @@
 RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
-    pip install "platform-plugin-aspects==v0.10.0"
+    pip install "platform-plugin-aspects==v0.11.0"
 
 RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
     pip install "edx-event-routing-backends>=9.3.0,<9.4"

tutoraspects/patches/openedx-dockerfile-post-python-requirements

@@ -1,5 +1,5 @@
 RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
-    pip install "platform-plugin-aspects==v0.10.0"
+    pip install "platform-plugin-aspects==v0.11.0"
 
 RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
     pip install "edx-event-routing-backends>=9.3.0,<9.4"

tutoraspects/plugin.py

@@ -40,7 +40,7 @@
         ("DOCKER_IMAGE_VECTOR", "timberio/vector:0.30.0-alpine"),
         (
             "EVENT_SINK_MODELS_ENABLED",
-            ["course_overviews"],
+            ["course_overviews", "tag", "taxonomy", "object_tag"],
         ),
         (
             "EVENT_SINK_PII_MODELS",
@@ -376,7 +376,7 @@
         # For now we are pulling this from github, which should allow maximum
         # flexibility for forking, running branches, specific versions, etc.
         ("DBT_REPOSITORY", "https://github.com/openedx/aspects-dbt"),
-        ("DBT_BRANCH", "v3.30.0"),
+        ("DBT_BRANCH", "v3.31.0"),
         ("DBT_SSH_KEY", ""),
         ("DBT_STATE_DIR", "/app/aspects-dbt/state"),
         ("DBT_PROFILES_DIR", "/app/aspects/dbt/"),

tutoraspects/templates/aspects/apps/aspects/dbt/profiles.yml

@@ -31,6 +31,8 @@ aspects: # this needs to match the profile in your dbt_project.yml file
         database_atomic_delay_before_drop_table_sec: 0
         # Make sure all drops and detaches complete before continuing
         database_atomic_wait_for_drop_and_detach_synchronously: 1
+        # Allow to drop/rename tables that have dependants. dbt-clickhouse use it for view backups
+        check_table_dependencies: 0
 
 
       {{ patch("dbt-profiles") | indent(6)}}

tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0039_event_sink_tag_taxonomy.py

@@ -0,0 +1,87 @@
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0039"
+down_revision = "0038"
+branch_labels = None
+depends_on = None
+on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else ""
+engine = (
+    "ReplicatedReplacingMergeTree"
+    if "{{CLICKHOUSE_CLUSTER_NAME}}"
+    else "ReplacingMergeTree"
+)
+
+
+def upgrade():
+    op.execute(
+        f"""
+        CREATE TABLE IF NOT EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.tag
+        {on_cluster}
+        (
+            id Int32,
+            taxonomy Int32,
+            parent Int32,
+            value String,
+            external_id String,
+            lineage String,
+            dump_id UUID NOT NULL,
+            time_last_dumped String NOT NULL
+        ) ENGINE {engine}
+        ORDER BY (id, time_last_dumped)
+        PRIMARY KEY (id, time_last_dumped);
+        """
+    )
+
+    op.execute(
+        f"""
+        CREATE TABLE IF NOT EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.taxonomy
+        {on_cluster}
+        (
+            id Int32,
+            name String,
+            dump_id UUID NOT NULL,
+            time_last_dumped String NOT NULL
+        ) ENGINE {engine}
+        ORDER BY (id, time_last_dumped)
+        PRIMARY KEY (id, time_last_dumped);
+        """
+    )
+
+
+    op.execute(
+        f"""
+        CREATE TABLE IF NOT EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.object_tag
+        {on_cluster}
+        (
+            id Int32,
+            object_id String,
+            taxonomy Int32,
+            tag Int32,
+            _value String,
+            _export_id String,
+            lineage String,
+            dump_id UUID NOT NULL,
+            time_last_dumped String NOT NULL
+        ) ENGINE {engine}
+        ORDER BY (id, time_last_dumped)
+        PRIMARY KEY (id, time_last_dumped);
+        """
+    )
+
+
+def downgrade():
+    op.execute(
+        "DROP TABLE IF EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.tag"
+        f"{on_cluster};"
+    )
+
+    op.execute(
+        "DROP TABLE IF EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.taxonomy"
+        f"{on_cluster};"
+    )
+
+    op.execute(
+        "DROP TABLE IF EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.object_tag"
+        f"{on_cluster};"
+    )

tutoraspects/templates/aspects/apps/aspects/scripts/bootstrap.sh

@@ -16,7 +16,7 @@ fi
 
 export branch=$(git -C aspects-dbt/ branch --show-current)
 export repo=$(git -C aspects-dbt/ config --get remote.origin.url)
-if [ "$DBT_BRANCH" != "$branch"  ] && [ "$DBT_REPOSITORY" != "$repo" ];
+if [ "$DBT_BRANCH" != "$branch"  ] || [ "$DBT_REPOSITORY" != "$repo" ];
 then
   rm -rf aspects-dbt