feat: add do command to update the authentication plugin of MySQL users

changelog.d/20240718_171945_danyal.faheem_mysql_authentication_plugin_change.md

@@ -0,0 +1 @@
+- [Improvement] Add a do command to update the authentication plugin of existing MySQL users from mysql_native_password to caching_sha2_password for compatibility with MySQL v8.4.0 and above. (by @Danyal-Faheem)

docs/local.rst

@@ -168,6 +168,27 @@ By default, only the tables in the openedx database are changed. For upgrading t
 
     tutor local do convert-mysql-utf8mb4-charset --database=discovery
 
+.. _update_mysql_authentication_plugin:
+
+Updating the authentication plugin of MySQL users
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As of MySQL v8.4.0, the ``mysql_native_password`` authentication plugin has been deprecated. Users created with this authentication plugin should ideally be updated to use the latest ``caching_sha2_password`` authentication plugin.
+
+Tutor makes it easy do so with this handy command::
+
+    tutor local do update-mysql-authentication-plugin USERNAME
+
+The password will not be required for official plugins that have database users as tutor can infer it from the config. If the password cannot be found by tutor, you will be prompted to enter the password interactively. Alternatively, the password can also be provided as an option::
+
+    tutor local do update-mysql-authentication-plugin USERNAME --password=PASSWORD
+
+To update the database users for a vanilla tutor installation::
+
+    tutor local do update-mysql-authentication-plugin $(tutor config printvalue OPENEDX_MYSQL_USERNAME)
+    tutor local do update-mysql-authentication-plugin $(tutor config printvalue MYSQL_ROOT_USERNAME)
+
+
 Running arbitrary ``manage.py`` commands
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/troubleshooting.rst

@@ -216,3 +216,13 @@ NPM Dependency Conflict When overriding ``@edx/frontend-component-header`` or ``
 ----------------------------------------------------------------------------------------------------------------
 
 The detailed steps are mentioned in `tutor-mfe <https://github.com/overhangio/tutor-mfe?tab=readme-ov-file#npm-dependency-conflict-when-overriding-edxfrontend-component-header-or-edxfrontend-component-footer>`__ documentation.
+
+"Plugin 'mysql_native_password' is not loaded"
+----------------------------------------------
+
+This issue can occur when Tutor is upgraded from v15 (Olive) or earlier to v18 (Redwood) or later because the users created in Tutor v15 and earlier utilize the mysql_native_password authentication plugin by default. This plugin has been deprecated as of MySQL v8.4.0 which is the default MySQL server used in Tutor v18 and onwards.
+
+The handy :ref:`update-mysql-authentication-plugin <update_mysql_authentication_plugin>` do command in tutor can be used to fix this issue. To update the database users for a vanilla tutor installation::
+
+    tutor local do update-mysql-authentication-plugin $(tutor config printvalue OPENEDX_MYSQL_USERNAME)
+    tutor local do update-mysql-authentication-plugin $(tutor config printvalue MYSQL_ROOT_USERNAME)

tests/commands/test_jobs.py

@@ -165,3 +165,47 @@ def test_convert_mysql_utf8mb4_charset_exclude_tables(self) -> None:
             self.assertIn("NOT", dc_args[-1])
             self.assertIn("course", dc_args[-1])
             self.assertIn("auth", dc_args[-1])
+
+    def test_update_mysql_authentication_plugin_official_plugin(self) -> None:
+        with temporary_root() as root:
+            self.invoke_in_root(root, ["config", "save"])
+            with patch("tutor.utils.docker_compose") as mock_docker_compose:
+                result = self.invoke_in_root(
+                    root,
+                    [
+                        "local",
+                        "do",
+                        "update-mysql-authentication-plugin",
+                        "openedx",
+                    ],
+                )
+                dc_args, _dc_kwargs = mock_docker_compose.call_args
+
+            self.assertIsNone(result.exception)
+            self.assertEqual(0, result.exit_code)
+            self.assertIn("lms-job", dc_args)
+            self.assertIn("caching_sha2_password", dc_args[-1])
+            self.assertIn("openedx", dc_args[-1])
+
+    def test_update_mysql_authentication_plugin_custom_plugin(self) -> None:
+        with temporary_root() as root:
+            self.invoke_in_root(root, ["config", "save"])
+            with patch("tutor.utils.docker_compose") as mock_docker_compose:
+                result = self.invoke_in_root(
+                    root,
+                    [
+                        "local",
+                        "do",
+                        "update-mysql-authentication-plugin",
+                        "mypluginuser",
+                        "--password=mypluginpassword",
+                    ],
+                )
+                dc_args, _dc_kwargs = mock_docker_compose.call_args
+
+            self.assertIsNone(result.exception)
+            self.assertEqual(0, result.exit_code)
+            self.assertIn("lms-job", dc_args)
+            self.assertIn("caching_sha2_password", dc_args[-1])
+            self.assertIn("mypluginuser", dc_args[-1])
+            self.assertIn("mypluginpassword", dc_args[-1])

tutor/commands/jobs.py

@@ -14,7 +14,11 @@
 from tutor import config as tutor_config
 from tutor import env, fmt, hooks
 from tutor.commands.context import Context
-from tutor.commands.jobs_utils import get_mysql_change_charset_query
+from tutor.commands.jobs_utils import (
+    create_user_template,
+    get_mysql_change_charset_query,
+    set_theme_template,
+)
 from tutor.hooks import priorities
 
 
@@ -109,24 +113,6 @@ def createuser(
     yield ("lms", create_user_template(superuser, staff, name, email, password))
 
 
-def create_user_template(
-    superuser: str, staff: bool, username: str, email: str, password: str
-) -> str:
-    opts = ""
-    if superuser:
-        opts += " --superuser"
-    if staff:
-        opts += " --staff"
-    return f"""
-./manage.py lms manage_user {opts} {username} {email}
-./manage.py lms shell -c "
-from django.contrib.auth import get_user_model
-u = get_user_model().objects.get(username='{username}')
-u.set_password('{password}')
-u.save()"
-"""
-
-
 @click.command(help="Import the demo course")
 @click.option(
     "-r",
@@ -273,43 +259,6 @@ def settheme(domains: list[str], theme_name: str) -> t.Iterable[tuple[str, str]]
     yield ("lms", set_theme_template(theme_name, domains))
 
 
-def set_theme_template(theme_name: str, domain_names: list[str]) -> str:
-    """
-    For each domain, get or create a Site object and assign the selected theme.
-    """
-    # Note that there are no double quotes " in this piece of code
-    python_command = """
-import sys
-from django.contrib.sites.models import Site
-def assign_theme(name, domain):
-    print('Assigning theme', name, 'to', domain)
-    if len(domain) > 50:
-            sys.stderr.write(
-                'Assigning a theme to a site with a long (> 50 characters) domain name.'
-                ' The displayed site name will be truncated to 50 characters.\\n'
-            )
-    site, _ = Site.objects.get_or_create(domain=domain)
-    if not site.name:
-        name_max_length = Site._meta.get_field('name').max_length
-        site.name = domain[:name_max_length]
-        site.save()
-    site.themes.all().delete()
-    if name != 'default':
-        site.themes.create(theme_dir_name=name)
-"""
-    domain_names = domain_names or [
-        "{{ LMS_HOST }}",
-        "{{ LMS_HOST }}:8000",
-        "{{ CMS_HOST }}",
-        "{{ CMS_HOST }}:8001",
-        "{{ PREVIEW_LMS_HOST }}",
-        "{{ PREVIEW_LMS_HOST }}:8000",
-    ]
-    for domain_name in domain_names:
-        python_command += f"assign_theme('{theme_name}', '{domain_name}')\n"
-    return f'./manage.py lms shell -c "{python_command}"'
-
-
 @click.command(context_settings={"ignore_unknown_options": True})
 @click.argument("args", nargs=-1)
 def sqlshell(args: list[str]) -> t.Iterable[tuple[str, str]]:
@@ -428,6 +377,95 @@ def generate_query_to_append(tables: list[str], exclude: bool = False) -> str:
     fmt.echo_info("MySQL charset and collation successfully upgraded")
 
 
+@click.command(
+    short_help="Update the authentication plugin of a mysql user to caching_sha2_password.",
+    help=(
+        "Update the authentication plugin of a mysql user to caching_sha2_password from mysql_native_password. You can specify either specific users to update or all to update all users."
+    ),
+)
+@click.option(
+    "-p",
+    "--password",
+    help="Specify password from the command line.",
+)
+@click.argument(
+    "user",
+)
+@click.pass_obj
+def update_mysql_authentication_plugin(
+    context: Context, user: str, password: str
+) -> t.Iterable[tuple[str, str]]:
+    """
+    Update the authentication plugin of MySQL users from mysql_native_password to caching_sha2_password
+    Handy command utilized when upgrading to v8.4 of MySQL which deprecates mysql_native_password
+    """
+
+    config = tutor_config.load(context.root)
+
+    if not config["RUN_MYSQL"]:
+        fmt.echo_info(
+            "You are not running MySQL (RUN_MYSQL=False). It is your "
+            "responsibility to update the authentication plugin of mysql users."
+        )
+        return
+
+    # Official plugins that have their own mysql user
+    known_plugins_with_mysql_keys = [
+        "credentials",
+        "discovery",
+        "ecommerce",
+        "jupyter",
+        "notes",
+        "openedx",
+        "xqueue",
+    ]
+
+    # Create a list of the usernames and passwords
+    known_mysql_credentials_keys = [
+        (f"{plugin.upper()}_MYSQL_USERNAME", f"{plugin.upper()}_MYSQL_PASSWORD")
+        for plugin in known_plugins_with_mysql_keys
+    ]
+    # Add the root user as it is the only one that is different from the rest
+    known_mysql_credentials_keys.append(("MYSQL_ROOT_USERNAME", "MYSQL_ROOT_PASSWORD"))
+
+    known_mysql_credentials = {}
+    # Build the dictionary of known credentials
+    for k, v in known_mysql_credentials_keys:
+        if username := config.get(k):
+            known_mysql_credentials[username] = config[v]
+
+    if not password:
+        password = known_mysql_credentials.get(user)  # type: ignore
+
+    # Prompt the user if password was not found in config
+    if not password:
+        password = click.prompt(
+            f"Please enter the password for the user {user}",
+            type=str,
+        )
+
+    host = "%"
+
+    query = f"ALTER USER IF EXISTS '{user}'@'{host}' IDENTIFIED with caching_sha2_password BY '{password}';"
+
+    yield (
+        "lms",
+        shlex.join(
+            [
+                "mysql",
+                "--user={{ MYSQL_ROOT_USERNAME }}",
+                "--password={{ MYSQL_ROOT_PASSWORD }}",
+                "--host={{ MYSQL_HOST }}",
+                "--port={{ MYSQL_PORT }}",
+                "--database={{ OPENEDX_MYSQL_DATABASE }}",
+                "--show-warnings",
+                "-e",
+                query,
+            ]
+        ),
+    )
+
+
 def add_job_commands(do_command_group: click.Group) -> None:
     """
     This is meant to be called with the `local/dev/k8s do` group commands, to add the
@@ -511,5 +549,6 @@ def do_callback(service_commands: t.Iterable[tuple[str, str]]) -> None:
         print_edx_platform_setting,
         settheme,
         sqlshell,
+        update_mysql_authentication_plugin,
     ]
 )

tutor/commands/jobs_utils.py

@@ -1,8 +1,24 @@
-"""
-This module provides utility methods for tutor `do` commands
+from __future__ import annotations
+
 
-Methods:
-- `get_mysql_change_charset_query`: Generates MySQL queries to upgrade the charset and collation of columns, tables, and databases.
+def create_user_template(
+    superuser: str, staff: bool, username: str, email: str, password: str
+) -> str:
+    """
+    Helper utility to generate the necessary commands to create a user in openedx
+    """
+    opts = ""
+    if superuser:
+        opts += " --superuser"
+    if staff:
+        opts += " --staff"
+    return f"""
+./manage.py lms manage_user {opts} {username} {email}
+./manage.py lms shell -c "
+from django.contrib.auth import get_user_model
+u = get_user_model().objects.get(username='{username}')
+u.set_password('{password}')
+u.save()"
 """
 
 
@@ -130,3 +146,40 @@ def get_mysql_change_charset_query(
             CALL UpdateColumns();
             CALL UpdateTables();
             """
+
+
+def set_theme_template(theme_name: str, domain_names: list[str]) -> str:
+    """
+    For each domain, get or create a Site object and assign the selected theme.
+    """
+    # Note that there are no double quotes " in this piece of code
+    python_command = """
+import sys
+from django.contrib.sites.models import Site
+def assign_theme(name, domain):
+    print('Assigning theme', name, 'to', domain)
+    if len(domain) > 50:
+            sys.stderr.write(
+                'Assigning a theme to a site with a long (> 50 characters) domain name.'
+                ' The displayed site name will be truncated to 50 characters.\\n'
+            )
+    site, _ = Site.objects.get_or_create(domain=domain)
+    if not site.name:
+        name_max_length = Site._meta.get_field('name').max_length
+        site.name = domain[:name_max_length]
+        site.save()
+    site.themes.all().delete()
+    if name != 'default':
+        site.themes.create(theme_dir_name=name)
+"""
+    domain_names = domain_names or [
+        "{{ LMS_HOST }}",
+        "{{ LMS_HOST }}:8000",
+        "{{ CMS_HOST }}",
+        "{{ CMS_HOST }}:8001",
+        "{{ PREVIEW_LMS_HOST }}",
+        "{{ PREVIEW_LMS_HOST }}:8000",
+    ]
+    for domain_name in domain_names:
+        python_command += f"assign_theme('{theme_name}', '{domain_name}')\n"
+    return f'./manage.py lms shell -c "{python_command}"'