Skip to content

Commit b043fa1

Browse files
Ian2012Ian2012
authored
Merge pull request #550 from openedx/bmtcril/upgrade_db_load
chore: Upgrade xapi-db-load to 1.0
2 parents 491df1c + f9b0e01 commit b043fa1

File tree

7 files changed

+91
-30
lines changed

7 files changed

+91
-30
lines changed

‎.github/workflows/test-tutor-aspects.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ jobs:
6060
- name: Test DBT
6161
run: tutor local do dbt -c "test"
6262
- name: Load test
63-
run: tutor local do load-xapi-test-data --num_batches=5 --batch_size=5
63+
run: tutor local do load-xapi-test-data
6464
- name: Import demo course
6565
run: tutor local do importdemocourse
6666
- name: Test commands
@@ -114,7 +114,7 @@ jobs:
114114
- name: Test DBT
115115
run: tutor dev do dbt -c "test"
116116
- name: Load test
117-
run: tutor dev do load-xapi-test-data --num_batches=5 --batch_size=5
117+
run: tutor dev do load-xapi-test-data
118118
- name: Import demo course
119119
run: tutor dev do importdemocourse
120120
- name: Test commands
@@ -186,7 +186,7 @@ jobs:
186186
- name: Test DBT
187187
run: tutor k8s do dbt -c "test"
188188
- name: Load test
189-
run: tutor k8s do load-xapi-test-data --num_batches=5 --batch_size=5
189+
run: tutor k8s do load-xapi-test-data
190190
- name: Import demo course
191191
run: tutor k8s do importdemocourse
192192
- name: Test commands

‎tutoraspects/commands_v0.py

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,19 +39,18 @@ def dbt(context, command) -> None:
3939

4040

4141
@click.command(help="Load generated fake xAPI test data to ClickHouse.")
42-
@click.option("-n", "--num_batches", default=100)
43-
@click.option("-s", "--batch_size", default=100)
42+
@click.option("-c", "--config_file", default="./xapi-db-load-config.yaml")
4443
@click.pass_obj
45-
def load_xapi_test_data(context, num_batches, batch_size) -> None:
44+
def load_xapi_test_data(context, config_file) -> None:
4645
"""
4746
Job that loads bogus test xAPI data to ClickHouse via Ralph.
4847
"""
4948
config = tutor_config.load(context.root)
5049
runner = context.job_runner(config)
5150

52-
command = f"""echo 'Making demo xapi script executable...'
53-
echo 'Done. Running script...'
54-
bash /app/aspects/scripts/clickhouse-demo-xapi-data.sh {num_batches} {batch_size}
51+
command = f"""echo 'Running script...'
52+
cd /app/aspects/scripts/
53+
bash clickhouse-demo-xapi-data.sh {config_file}
5554
echo 'Done!';
5655
"""
5756
runner.run_job("aspects", command)
@@ -87,9 +86,8 @@ def alembic(context, command) -> None:
8786
runner = context.job_runner(config)
8887

8988
command = f"""echo 'Making demo xapi script executable...'
90-
echo 'Done. Running script...'
89+
echo 'Running script...'
9190
bash /app/aspects/scripts/alembic.sh {command}
92-
echo 'Done!';
9391
"""
9492
runner.run_job("aspects", command)
9593

‎tutoraspects/commands_v1.py

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,17 @@
1717

1818

1919
@click.command()
20-
@click.option("-n", "--num_batches", default=100)
21-
@click.option("-s", "--batch_size", default=100)
22-
def load_xapi_test_data(num_batches: int, batch_size: int) -> list[tuple[str, str]]:
20+
@click.option("-c", "--config_file", default="./xapi-db-load-config.yaml")
21+
def load_xapi_test_data(config_file) -> list[tuple[str, str]]:
2322
"""
2423
Job that loads bogus test xAPI data to ClickHouse via Ralph.
2524
"""
2625
return [
2726
(
2827
"aspects",
29-
"echo 'Making demo xapi script executable...' && "
30-
"echo 'Done. Running script...' && "
31-
f"bash /app/aspects/scripts/clickhouse-demo-xapi-data.sh {num_batches}"
32-
f" {batch_size} && "
28+
"echo 'Running script...' && "
29+
"cd /app/aspects/scripts/ && "
30+
f"bash clickhouse-demo-xapi-data.sh {config_file} && "
3331
"echo 'Done!';",
3432
),
3533
]

‎tutoraspects/patches/xapi-db-load-config-yaml

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
# Ralph / ClickHouse backend configuration
2+
# ########################################
3+
backend: ralph_clickhouse
4+
db_host: {{ CLICKHOUSE_HOST }}
5+
db_port: {{ CLICKHOUSE_INTERNAL_HTTP_PORT }}
6+
db_name: {{ ASPECTS_XAPI_DATABASE }}
7+
db_username: {{ CLICKHOUSE_ADMIN_USER }}
8+
db_password: {{ CLICKHOUSE_ADMIN_PASSWORD }}
9+
lrs_url: "{% if RUN_RALPH %}http://ralph:{{ RALPH_PORT }}{% else %}{% if RALPH_RUN_HTTPS %}https://{% else %}http://{% endif %}{{ RALPH_HOST }}{% endif %}/xAPI/statements/"
10+
lrs_username: "{{RALPH_LMS_USERNAME}}"
11+
lrs_password: "{{RALPH_LMS_PASSWORD}}"
12+
13+
# Run options
14+
log_dir:
15+
num_batches: 3
16+
batch_size: 100
17+
18+
# Overall start and end date for the entire run
19+
start_date: 2014-01-01
20+
end_date: 2023-11-27
21+
22+
# All courses will be this long, and be fit into the start / end dates
23+
# This must be less than end_date - start_date days.
24+
course_length_days: 120
25+
26+
# The size of the test
27+
num_organizations: 3
28+
num_actors: 10
29+
30+
# How many of each size course to create. The sum of these is the total number
31+
# of courses created for the test.
32+
num_course_sizes:
33+
small: 1
34+
medium: 1
35+
large: 1
36+
huge: 1
37+
38+
# Course size configurations, how many of each type of object are created for
39+
# each course of this size. "actors" must be less than or equal to "num_actors".
40+
# For a course of this size to be created it needs to exist both here and in
41+
# "num_course_sizes".
42+
course_size_makeup:
43+
small:
44+
actors: 5
45+
problems: 20
46+
videos: 10
47+
chapters: 3
48+
sequences: 10
49+
verticals: 20
50+
forum_posts: 20
51+
medium:
52+
actors: 7
53+
problems: 40
54+
videos: 20
55+
chapters: 4
56+
sequences: 20
57+
verticals: 30
58+
forum_posts: 40
59+
large:
60+
actors: 10
61+
problems: 80
62+
videos: 30
63+
chapters: 5
64+
sequences: 40
65+
verticals: 80
66+
forum_posts: 200
67+
huge:
68+
actors: 10
69+
problems: 160
70+
videos: 40
71+
chapters: 10
72+
sequences: 50
73+
verticals: 100
74+
forum_posts: 1000

‎tutoraspects/templates/aspects/apps/aspects/scripts/clickhouse-demo-xapi-data.sh

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,5 @@
11
#!/usr/bin/env bash
22
echo "Loading demo xAPI data..."
3-
xapi-db-load --backend ralph_clickhouse \
4-
--num_batches $1 \
5-
--batch_size $2 \
6-
--db_host "{{ CLICKHOUSE_HOST }}" \
7-
--db_port "{{ CLICKHOUSE_INTERNAL_HTTP_PORT }}" \
8-
--db_username "{{ CLICKHOUSE_ADMIN_USER }}" \
9-
--db_password "{{ CLICKHOUSE_ADMIN_PASSWORD }}" \
10-
--db_name "{{ ASPECTS_XAPI_DATABASE }}" \
11-
--lrs_url "{% if RUN_RALPH %}http://ralph:{{ RALPH_PORT }}{% else %}{% if RALPH_RUN_HTTPS %}https://{% else %}http://{% endif %}{{ RALPH_HOST }}{% endif %}/xAPI/statements/" \
12-
--lrs_username "{{RALPH_LMS_USERNAME}}" \
13-
--lrs_password "{{RALPH_LMS_PASSWORD}}"
3+
xapi-db-load load-db --config_file $1
144

155
echo "Done."

‎tutoraspects/templates/aspects/apps/aspects/scripts/xapi-db-load-config.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{{ patch('xapi-db-load-config-yaml') }}

‎tutoraspects/templates/aspects/build/aspects/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@ clickhouse-sqlalchemy==0.1.9
44
# dbt packages
55
dbt-core==1.7.0
66
dbt-clickhouse==1.7.1
7-
git+https://github.com/openedx/xapi-db-load@0.8#egg=xapi-db-load==0.8
7+
git+https://github.com/openedx/xapi-db-load@1.0#egg=xapi-db-load==1.0

0 commit comments

Comments
 (0)