Merge pull request openedx#18864 from open-craft/paulo/export-import-…

…library

[OC-3682] Implement management commands to export/import content libraries

cms/djangoapps/contentstore/management/commands/export_content_library.py

@@ -0,0 +1,65 @@
+"""
+Script for exporting a content library from Mongo to a tar.gz file
+"""
+from __future__ import print_function
+import os
+import shutil
+
+from django.core.management.base import BaseCommand, CommandError
+from opaque_keys import InvalidKeyError
+from opaque_keys.edx.keys import CourseKey
+from opaque_keys.edx.locator import LibraryLocator
+from xmodule.modulestore.django import modulestore
+
+from cms.djangoapps.contentstore import tasks
+
+
+class Command(BaseCommand):
+    """
+    Export the specified content library into a directory.  Output will need to be tar zxcf'ed.
+    """
+    help = 'Export the specified content library into a directory'
+
+    def add_arguments(self, parser):
+        parser.add_argument('library_id')
+        parser.add_argument('output_path', nargs='?')
+
+    def handle(self, *args, **options):
+        """
+        Given a content library id, and an output_path folder.  Export the
+        corresponding course from mongo and put it directly in the folder.
+        """
+        module_store = modulestore()
+        try:
+            library_key = CourseKey.from_string(options['library_id'])
+        except InvalidKeyError:
+            raise CommandError(u'Invalid library ID: "{0}".'.format(options['library_id']))
+        if not isinstance(library_key, LibraryLocator):
+            raise CommandError(u'Argument "{0}" is not a library key'.format(options['library_id']))
+
+        library = module_store.get_library(library_key)
+        if library is None:
+            raise CommandError(u'Library "{0}" not found.'.format(options['library_id']))
+
+        dest_path = options['output_path'] or '.'
+        if not os.path.isdir(dest_path):
+            raise CommandError(u'Output path "{0}" not found.'.format(dest_path))
+
+        try:
+            # Generate archive using the handy tasks implementation
+            tarball = tasks.create_export_tarball(library, library_key, {}, None)
+        except Exception as e:
+            raise CommandError(u'Failed to export "{0}" with "{1}"'.format(library_key, e))
+        else:
+            with tarball:
+                # Save generated archive with keyed filename
+                prefix, suffix, n = str(library_key).replace(':', '+'), '.tar.gz', 0
+                while os.path.exists(prefix + suffix):
+                    n += 1
+                    prefix = u'{0}_{1}'.format(prefix.rsplit('_', 1)[0], n) if n > 1 else u'{}_1'.format(prefix)
+                filename = prefix + suffix
+                target = os.path.join(dest_path, filename)
+                tarball.file.seek(0)
+                with open(target, 'w') as f:
+                    shutil.copyfileobj(tarball.file, f)
+                print(u'Library "{0}" exported to "{1}"'.format(library.location.library_key, target))

cms/djangoapps/contentstore/management/commands/import_content_library.py

@@ -0,0 +1,122 @@
+"""
+Script for importing a content library from a tar.gz file
+"""
+from __future__ import print_function
+
+import base64
+import os
+import tarfile
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.core.exceptions import SuspiciousOperation
+from django.core.management.base import BaseCommand, CommandError
+from lxml import etree
+from opaque_keys.edx.locator import LibraryLocator
+from path import Path
+from xmodule.contentstore.django import contentstore
+from xmodule.modulestore import ModuleStoreEnum
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.exceptions import DuplicateCourseError
+from xmodule.modulestore.xml_importer import import_library_from_xml
+
+from cms.djangoapps.contentstore.utils import add_instructor
+from openedx.core.lib.extract_tar import safetar_extractall
+
+
+class Command(BaseCommand):
+    """
+    Import the specified content library archive.
+    """
+    help = 'Import the specified content library into mongo'
+
+    def add_arguments(self, parser):
+        parser.add_argument('archive_path')
+        parser.add_argument('owner_username')
+
+    def handle(self, *args, **options):
+        """
+        Given a content library archive path, import the corresponding course to mongo.
+        """
+
+        archive_path = options['archive_path']
+        username = options['owner_username']
+
+        data_root = Path(settings.GITHUB_REPO_ROOT)
+        subdir = base64.urlsafe_b64encode(os.path.basename(archive_path))
+        course_dir = data_root / subdir
+
+        # Extract library archive
+        tar_file = tarfile.open(archive_path)
+        try:
+            safetar_extractall(tar_file, course_dir.encode('utf-8'))
+        except SuspiciousOperation as exc:
+            raise CommandError(u'\n=== Course import {0}: Unsafe tar file - {1}\n'.format(archive_path, exc.args[0]))
+        finally:
+            tar_file.close()
+
+        # Paths to the library.xml file
+        abs_xml_path = os.path.join(course_dir, 'library')
+        rel_xml_path = os.path.relpath(abs_xml_path, data_root)
+
+        # Gather library metadata from XML file
+        xml_root = etree.parse(abs_xml_path / 'library.xml').getroot()
+        if xml_root.tag != 'library':
+            raise CommandError(u'Failed to import {0}: Not a library archive'.format(archive_path))
+
+        metadata = xml_root.attrib
+        org = metadata['org']
+        library = metadata['library']
+        display_name = metadata['display_name']
+
+        # Fetch user and library key
+        user = User.objects.get(username=username)
+        courselike_key, created = _get_or_create_library(org, library, display_name, user)
+
+        # Check if data would be overwritten
+        ans = ''
+        while not created and ans not in ['y', 'yes', 'n', 'no']:
+            inp = raw_input(u'Library "{0}" already exists, overwrite it? [y/n] '.format(courselike_key))
+            ans = inp.lower()
+        if ans.startswith('n'):
+            print(u'Aborting import of "{0}"'.format(courselike_key))
+            return
+
+        # At last, import the library
+        try:
+            import_library_from_xml(
+                modulestore(), user.id,
+                settings.GITHUB_REPO_ROOT, [rel_xml_path],
+                load_error_modules=False,
+                static_content_store=contentstore(),
+                target_id=courselike_key
+            )
+        except Exception:
+            print(u'\n=== Failed to import library-v1:{0}+{1}'.format(org, library))
+            raise
+
+        print(u'Library "{0}" imported to "{1}"'.format(archive_path, courselike_key))
+
+
+def _get_or_create_library(org, number, display_name, user):
+    """
+    Create or retrieve given library and return its course-like key
+    """
+
+    try:
+        # Create library if it does not exist
+        store = modulestore()
+        with store.default_store(ModuleStoreEnum.Type.split):
+            library = store.create_library(
+                org=org,
+                library=number,
+                user_id=user.id,
+                fields={
+                    "display_name": display_name
+                },
+            )
+        add_instructor(library.location.library_key, user, user)
+        return library.location.library_key, True
+    except DuplicateCourseError:
+        # Course exists, return its key
+        return LibraryLocator(org=org, library=number), False