From 5d59e2df0b4c60d62341bd6b11bf69dbd72cbfe2 Mon Sep 17 00:00:00 2001 From: Paulo Viadanna Date: Wed, 22 Aug 2018 15:08:42 -0300 Subject: [PATCH 1/2] Content library export/import commands --- .../commands/export_content_library.py | 64 ++++++++++ .../commands/import_content_library.py | 119 ++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 cms/djangoapps/contentstore/management/commands/export_content_library.py create mode 100644 cms/djangoapps/contentstore/management/commands/import_content_library.py diff --git a/cms/djangoapps/contentstore/management/commands/export_content_library.py b/cms/djangoapps/contentstore/management/commands/export_content_library.py new file mode 100644 index 00000000000..76011a45cd0 --- /dev/null +++ b/cms/djangoapps/contentstore/management/commands/export_content_library.py @@ -0,0 +1,64 @@ +""" +Script for exporting a content library from Mongo to a tar.gz file +""" +from __future__ import print_function +import os + +from django.core.management.base import BaseCommand, CommandError +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locator import LibraryLocator +from xmodule.modulestore.django import modulestore + +from cms.djangoapps.contentstore import tasks + + +class Command(BaseCommand): + """ + Export the specified content library into a directory. Output will need to be tar zxcf'ed. + """ + help = 'Export the specified content library into a directory' + + def add_arguments(self, parser): + parser.add_argument('library_id') + parser.add_argument('output_path', nargs='?') + + def handle(self, *args, **options): + """ + Given a content library id, and an output_path folder. Export the + corresponding course from mongo and put it directly in the folder. + """ + module_store = modulestore() + try: + library_key = CourseKey.from_string(options['library_id']) + assert isinstance(library_key, LibraryLocator) + except InvalidKeyError: + raise CommandError(u'Invalid library ID: "{0}".'.format(options['library_id'])) + except AssertionError: + raise CommandError(u'Argument "{0}" is not a library key'.format(options['library_id'])) + + library = module_store.get_library(library_key) + if library is None: + raise CommandError(u'Library "{0}" not found.'.format(options['library_id'])) + + dest_path = options['output_path'] or '.' + if not os.path.isdir(dest_path): + raise CommandError(u'Output path "{0}" not found.'.format(dest_path)) + + try: + # Generate archive using the handy tasks implementation + tarball = tasks.create_export_tarball(library, library_key, {}, None) + except Exception as e: + raise CommandError(u'Failed to export "{0}" with "{1}"'.format(library_key, e)) + else: + # Save generated archive with keyed filename + prefix, suffix, n = str(library_key).replace(':', '+'), '.tar.gz', 0 + while os.path.exists(prefix + suffix): + n += 1 + prefix = u'{0}_{1}'.format(prefix.rsplit('_', 1)[0], n) if n > 1 else u'{}_1'.format(prefix) + filename = prefix + suffix + target = os.path.join(dest_path, filename) + tarball.file.seek(0) + with open(target, 'w') as f: + f.write(tarball.file.read()) + print(u'Library "{0}" exported to "{1}"'.format(library.location.library_key, target)) diff --git a/cms/djangoapps/contentstore/management/commands/import_content_library.py b/cms/djangoapps/contentstore/management/commands/import_content_library.py new file mode 100644 index 00000000000..b967932161b --- /dev/null +++ b/cms/djangoapps/contentstore/management/commands/import_content_library.py @@ -0,0 +1,119 @@ +""" +Script for importing a content library from a tar.gz file +""" +from __future__ import print_function + +import base64 +import os +import tarfile + +from django.conf import settings +from django.contrib.auth.models import User +from django.core.exceptions import SuspiciousOperation +from django.core.management.base import BaseCommand, CommandError +from lxml import etree +from opaque_keys.edx.keys import CourseKey +from path import Path +from xmodule.contentstore.django import contentstore +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.exceptions import DuplicateCourseError +from xmodule.modulestore.xml_importer import import_library_from_xml + +from cms.djangoapps.contentstore.utils import add_instructor +from openedx.core.lib.extract_tar import safetar_extractall + + +class Command(BaseCommand): + """ + Import the specified content library archive. + """ + help = 'Import the specified content library into mongo' + + def add_arguments(self, parser): + parser.add_argument('archive_path') + parser.add_argument('owner_username') + + def handle(self, *args, **options): + """ + Given a content library archive path, import the corresponding course to mongo. + """ + + archive_path = options['archive_path'] + username = options['owner_username'] + + data_root = Path(settings.GITHUB_REPO_ROOT) + subdir = base64.urlsafe_b64encode(os.path.basename(archive_path)) + course_dir = data_root / subdir + + # Extract library archive + tar_file = tarfile.open(archive_path) + try: + safetar_extractall(tar_file, course_dir.encode('utf-8')) + except SuspiciousOperation as exc: + raise CommandError(u'\n=== Course import {0}: Unsafe tar file - {1}\n'.format(archive_path, exc.args[0])) + finally: + tar_file.close() + + # Paths to the library.xml file + abs_xml_path = os.path.join(course_dir, 'library') + rel_xml_path = os.path.relpath(abs_xml_path, data_root) + + # Gather library metadata from XML file + xml_root = etree.parse(abs_xml_path / 'library.xml').getroot() + assert xml_root.tag == 'library' + metadata = xml_root.attrib + org = metadata['org'] + library = metadata['library'] + display_name = metadata['display_name'] + + # Fetch user and library key + user = User.objects.get(username=username) + courselike_key, created = _get_or_create_library(org, library, display_name, user) + + # Check if data would be overwritten + ans = '' + while not created and ans.lower() not in ['y', 'yes', 'n', 'no']: + ans = raw_input(u'Library "{0}" already exists, overwrite it? [y/n] '.format(courselike_key)) + if ans.startswith('n'): + print(u'Aborting import of "{0}"'.format(courselike_key)) + return + + # At last, import the library + try: + import_library_from_xml( + modulestore(), user.id, + settings.GITHUB_REPO_ROOT, [rel_xml_path], + load_error_modules=False, + static_content_store=contentstore(), + target_id=courselike_key + ) + except Exception as e: + print(u'\n=== Failed to import library-v1:{0}+{1}'.format(org, library)) + raise e + + print(u'Library "{0}" imported to "{1}"'.format(archive_path, courselike_key)) + + +def _get_or_create_library(org, number, display_name, user): + """ + Create or retrieve given library and return its course-like key + """ + + try: + # Create library if it does not exist + store = modulestore() + with store.default_store(ModuleStoreEnum.Type.split): + library = store.create_library( + org=org, + library=number, + user_id=user.id, + fields={ + "display_name": display_name + }, + ) + add_instructor(library.location.library_key, user, user) + return library.location.library_key, True + except DuplicateCourseError: + # Course exists, return its key + return CourseKey.from_string(u'library-v1:{0}+{1}'.format(org, number)), False From 754090279e515144852e5c2238062857959791f2 Mon Sep 17 00:00:00 2001 From: Matjaz Gregoric Date: Tue, 25 Sep 2018 10:07:09 +0200 Subject: [PATCH 2/2] Address review comments. --- .../commands/export_content_library.py | 27 ++++++++++--------- .../commands/import_content_library.py | 17 +++++++----- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/cms/djangoapps/contentstore/management/commands/export_content_library.py b/cms/djangoapps/contentstore/management/commands/export_content_library.py index 76011a45cd0..9c914485a1b 100644 --- a/cms/djangoapps/contentstore/management/commands/export_content_library.py +++ b/cms/djangoapps/contentstore/management/commands/export_content_library.py @@ -3,6 +3,7 @@ """ from __future__ import print_function import os +import shutil from django.core.management.base import BaseCommand, CommandError from opaque_keys import InvalidKeyError @@ -31,10 +32,9 @@ def handle(self, *args, **options): module_store = modulestore() try: library_key = CourseKey.from_string(options['library_id']) - assert isinstance(library_key, LibraryLocator) except InvalidKeyError: raise CommandError(u'Invalid library ID: "{0}".'.format(options['library_id'])) - except AssertionError: + if not isinstance(library_key, LibraryLocator): raise CommandError(u'Argument "{0}" is not a library key'.format(options['library_id'])) library = module_store.get_library(library_key) @@ -51,14 +51,15 @@ def handle(self, *args, **options): except Exception as e: raise CommandError(u'Failed to export "{0}" with "{1}"'.format(library_key, e)) else: - # Save generated archive with keyed filename - prefix, suffix, n = str(library_key).replace(':', '+'), '.tar.gz', 0 - while os.path.exists(prefix + suffix): - n += 1 - prefix = u'{0}_{1}'.format(prefix.rsplit('_', 1)[0], n) if n > 1 else u'{}_1'.format(prefix) - filename = prefix + suffix - target = os.path.join(dest_path, filename) - tarball.file.seek(0) - with open(target, 'w') as f: - f.write(tarball.file.read()) - print(u'Library "{0}" exported to "{1}"'.format(library.location.library_key, target)) + with tarball: + # Save generated archive with keyed filename + prefix, suffix, n = str(library_key).replace(':', '+'), '.tar.gz', 0 + while os.path.exists(prefix + suffix): + n += 1 + prefix = u'{0}_{1}'.format(prefix.rsplit('_', 1)[0], n) if n > 1 else u'{}_1'.format(prefix) + filename = prefix + suffix + target = os.path.join(dest_path, filename) + tarball.file.seek(0) + with open(target, 'w') as f: + shutil.copyfileobj(tarball.file, f) + print(u'Library "{0}" exported to "{1}"'.format(library.location.library_key, target)) diff --git a/cms/djangoapps/contentstore/management/commands/import_content_library.py b/cms/djangoapps/contentstore/management/commands/import_content_library.py index b967932161b..014fa37946b 100644 --- a/cms/djangoapps/contentstore/management/commands/import_content_library.py +++ b/cms/djangoapps/contentstore/management/commands/import_content_library.py @@ -12,7 +12,7 @@ from django.core.exceptions import SuspiciousOperation from django.core.management.base import BaseCommand, CommandError from lxml import etree -from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locator import LibraryLocator from path import Path from xmodule.contentstore.django import contentstore from xmodule.modulestore import ModuleStoreEnum @@ -61,7 +61,9 @@ def handle(self, *args, **options): # Gather library metadata from XML file xml_root = etree.parse(abs_xml_path / 'library.xml').getroot() - assert xml_root.tag == 'library' + if xml_root.tag != 'library': + raise CommandError(u'Failed to import {0}: Not a library archive'.format(archive_path)) + metadata = xml_root.attrib org = metadata['org'] library = metadata['library'] @@ -73,8 +75,9 @@ def handle(self, *args, **options): # Check if data would be overwritten ans = '' - while not created and ans.lower() not in ['y', 'yes', 'n', 'no']: - ans = raw_input(u'Library "{0}" already exists, overwrite it? [y/n] '.format(courselike_key)) + while not created and ans not in ['y', 'yes', 'n', 'no']: + inp = raw_input(u'Library "{0}" already exists, overwrite it? [y/n] '.format(courselike_key)) + ans = inp.lower() if ans.startswith('n'): print(u'Aborting import of "{0}"'.format(courselike_key)) return @@ -88,9 +91,9 @@ def handle(self, *args, **options): static_content_store=contentstore(), target_id=courselike_key ) - except Exception as e: + except Exception: print(u'\n=== Failed to import library-v1:{0}+{1}'.format(org, library)) - raise e + raise print(u'Library "{0}" imported to "{1}"'.format(archive_path, courselike_key)) @@ -116,4 +119,4 @@ def _get_or_create_library(org, number, display_name, user): return library.location.library_key, True except DuplicateCourseError: # Course exists, return its key - return CourseKey.from_string(u'library-v1:{0}+{1}'.format(org, number)), False + return LibraryLocator(org=org, library=number), False