From f4c2b466713166ba082dff4080a50a518d8b89dc Mon Sep 17 00:00:00 2001 From: Muhammad Adeel Tajamul <77053848+muhammadadeeltajamul@users.noreply.github.com> Date: Mon, 13 Jan 2025 12:56:40 +0500 Subject: [PATCH 01/18] chore: updated notification preference url in email digest (#36101) --- openedx/core/djangoapps/notifications/email/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/notifications/email/utils.py b/openedx/core/djangoapps/notifications/email/utils.py index 34c245308785..ad8b8b85dfd8 100644 --- a/openedx/core/djangoapps/notifications/email/utils.py +++ b/openedx/core/djangoapps/notifications/email/utils.py @@ -100,7 +100,7 @@ def create_email_template_context(username): "mailing_address": settings.CONTACT_MAILING_ADDRESS, "logo_url": get_logo_url_for_email(), "social_media": social_media_info, - "notification_settings_url": f"{settings.ACCOUNT_MICROFRONTEND_URL}/notifications", + "notification_settings_url": f"{settings.ACCOUNT_MICROFRONTEND_URL}/#notifications", "unsubscribe_url": get_unsubscribe_link(username, patch) } From cd9b90fc213a535487f909a07c094b18d5466cac Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Mon, 13 Jan 2025 23:39:47 +0530 Subject: [PATCH 02/18] feat: show math in plain text in library cards (#36055) Converts mathjax equations to unicode to be rendered as plain text in library card previews --- .../djangoapps/content/search/documents.py | 3 +- .../content/search/plain_text_math.py | 161 ++++++++++++++++++ .../content/search/tests/test_documents.py | 118 +++++++++++++ requirements/edx/base.txt | 2 + requirements/edx/development.txt | 4 + requirements/edx/doc.txt | 2 + requirements/edx/kernel.in | 1 + requirements/edx/testing.txt | 2 + 8 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 openedx/core/djangoapps/content/search/plain_text_math.py diff --git a/openedx/core/djangoapps/content/search/documents.py b/openedx/core/djangoapps/content/search/documents.py index 40fe4529272b..98cd7d576e0a 100644 --- a/openedx/core/djangoapps/content/search/documents.py +++ b/openedx/core/djangoapps/content/search/documents.py @@ -14,6 +14,7 @@ from rest_framework.exceptions import NotFound from openedx.core.djangoapps.content.search.models import SearchAccess +from openedx.core.djangoapps.content.search.plain_text_math import process_mathjax from openedx.core.djangoapps.content_libraries import api as lib_api from openedx.core.djangoapps.content_tagging import api as tagging_api from openedx.core.djangoapps.xblock import api as xblock_api @@ -220,7 +221,7 @@ class implementation returns only: # Generate description from the content description = _get_description_from_block_content(block_type, content_data) if description: - block_data[Fields.description] = description + block_data[Fields.description] = process_mathjax(description) except Exception as err: # pylint: disable=broad-except log.exception(f"Failed to process index_dictionary for {block.usage_key}: {err}") diff --git a/openedx/core/djangoapps/content/search/plain_text_math.py b/openedx/core/djangoapps/content/search/plain_text_math.py new file mode 100644 index 000000000000..70f6c3fd2cf0 --- /dev/null +++ b/openedx/core/djangoapps/content/search/plain_text_math.py @@ -0,0 +1,161 @@ +""" +Helper class to convert mathjax equations to plain text. +""" + +import re + +import unicodeit + + +class InvalidMathEquation(Exception): + """Raised when mathjax equation is invalid. This is used to skip all transformations.""" + + +class EqnPatternNotFound(Exception): + """Raised when a pattern is not found in equation. This is used to skip a specific transformation.""" + + +class PlainTextMath: + """ + Converts mathjax equations to plain text using unicodeit and some preprocessing. + """ + equation_pattern = re.compile( + r'\[mathjaxinline\](.*?)\[\/mathjaxinline\]|\[mathjax\](.*?)\[\/mathjax\]|\\\((.*?)\\\)|\\\[(.*?)\\\]' + ) + eqn_replacements = ( + # just remove prefix `\` + ("\\sin", "sin"), + ("\\cos", "cos"), + ("\\tan", "tan"), + ("\\arcsin", "arcsin"), + ("\\arccos", "arccos"), + ("\\arctan", "arctan"), + ("\\cot", "cot"), + ("\\sec", "sec"), + ("\\csc", "csc"), + # Is used for matching brackets in mathjax, should not be required in plain text. + ("\\left", ""), + ("\\right", ""), + ) + regex_replacements = ( + # Makes text bold, so not required in plain text. + (re.compile(r'{\\bf (.*?)}'), r"\1"), + ) + extract_inner_texts = ( + # Replaces any eqn: `\name{inner_text}` with `inner_text` + "\\mathbf{", + "\\bm{", + ) + frac_open_close_pattern = re.compile(r"}\s*{") + + @staticmethod + def _nested_bracket_matcher(equation: str, opening_pattern: str) -> str: + r""" + Matches opening and closing brackets in given string. + + Args: + equation: string + opening_pattern: for example, `\mathbf{` + + Returns: + String inside the eqn brackets + """ + start = equation.find(opening_pattern) + if start == -1: + raise EqnPatternNotFound() + open_count = 0 + inner_start = start + len(opening_pattern) + for i, char in enumerate(equation[inner_start:]): + if char == "{": + open_count += 1 + if char == "}": + if open_count == 0: + break + open_count -= 1 + else: + raise InvalidMathEquation() + # In below example `|` symbol is used to denote index position + # |\mathbf{, \mathbf{|, \mathbf{some_text|}, \mathbf{some_text}| + return (start, inner_start, inner_start + i, inner_start + i + 1) + + def _fraction_handler(self, equation: str) -> str: + r""" + Converts `\frac{x}{y}` to `(x/y)` while handling nested `{}`. + + For example: `\frac{2}{\sqrt{1+y}}` is converted to `(2/\sqrt{1+y})`. + + Args: + equation: string + + Returns: + String with `\frac` replaced by normal `/` symbol. + """ + try: + n_start, n_inner_start, n_inner_end, n_end = self._nested_bracket_matcher(equation, "\\frac{") + except EqnPatternNotFound: + return equation + + numerator = equation[n_inner_start:n_inner_end] + # Handle nested fractions + numerator = self._fraction_handler(numerator) + + try: + _, d_inner_start, d_inner_end, d_end = self._nested_bracket_matcher(equation[n_end:], "{") + except EqnPatternNotFound: + return equation + + denominator = equation[n_end + d_inner_start:n_end + d_inner_end] + # Handle nested fractions + denominator = self._fraction_handler(denominator) + # Now re-create the equation with `(numerator / denominator)` + equation = equation[:n_start] + f"({numerator}/{denominator})" + equation[n_end + d_end:] + return equation + + def _nested_text_extractor(self, equation: str, pattern: str) -> str: + """ + Recursively extracts text from equation for given pattern + """ + try: + start, inner_start, inner_end, end = self._nested_bracket_matcher(equation, pattern) + inner_text = equation[inner_start:inner_end] + inner_text = self._nested_text_extractor(inner_text, pattern) + equation = equation[:start] + inner_text + equation[end:] + except EqnPatternNotFound: + pass + return equation + + def _handle_replacements(self, equation: str) -> str: + """ + Makes a bunch of replacements in equation string. + """ + for q, replacement in self.eqn_replacements: + equation = equation.replace(q, replacement) + for pattern in self.extract_inner_texts: + equation = self._nested_text_extractor(equation, pattern) + for pattern, replacement in self.regex_replacements: + equation = re.sub(pattern, replacement, equation) + return equation + + def run(self, eqn_matches: re.Match) -> str: + """ + Takes re.Match object and runs conversion process on each match group. + """ + groups = eqn_matches.groups() + for group in groups: + if not group: + continue + original = group + try: + group = self._handle_replacements(group) + group = self._fraction_handler(group) + return unicodeit.replace(group) + except Exception: # pylint: disable=broad-except + return original + return None + + +processor = PlainTextMath() + + +def process_mathjax(content: str) -> str: + return re.sub(processor.equation_pattern, processor.run, content) diff --git a/openedx/core/djangoapps/content/search/tests/test_documents.py b/openedx/core/djangoapps/content/search/tests/test_documents.py index 603cc8d92f5e..a97caae168d6 100644 --- a/openedx/core/djangoapps/content/search/tests/test_documents.py +++ b/openedx/core/djangoapps/content/search/tests/test_documents.py @@ -477,3 +477,121 @@ def test_collection_with_published_library(self): "num_children": 1 } } + + def test_mathjax_plain_text_conversion_for_search(self): + """ + Test how an HTML block with mathjax equations gets converted to plain text in search description. + """ + # pylint: disable=line-too-long + eqns = [ + # (input, expected output) + ('Simple addition: \\( 2 + 3 \\)', 'Simple addition: 2 + 3'), + ('Simple subtraction: \\( 5 - 2 \\)', 'Simple subtraction: 5 − 2'), + ('Simple multiplication: \\( 4 * 6 \\)', 'Simple multiplication: 4 * 6'), + ('Simple division: \\( 8 / 2 \\)', 'Simple division: 8 / 2'), + ('Mixed arithmetic: \\( 2 + 3 4 \\)', 'Mixed arithmetic: 2 + 3 4'), + ('Simple exponentiation: \\[ 2^3 \\]', 'Simple exponentiation: 2³'), + ('Root extraction: \\[ 16^{1/2} \\]', 'Root extraction: 16¹^/²'), + ('Exponent with multiple terms: \\[ (2 + 3)^2 \\]', 'Exponent with multiple terms: (2 + 3)²'), + ('Nested exponents: \\[ 2^(3^2) \\]', 'Nested exponents: 2⁽3²)'), + ('Mixed roots: \\[ 8^{1/2} 3^2 \\]', 'Mixed roots: 8¹^/² 3²'), + ('Simple fraction: [mathjaxinline] 3/4 [/mathjaxinline]', 'Simple fraction: 3/4'), + ( + 'Decimal to fraction conversion: [mathjaxinline] 0.75 = 3/4 [/mathjaxinline]', + 'Decimal to fraction conversion: 0.75 = 3/4', + ), + ('Mixed fractions: [mathjaxinline] 1 1/2 = 3/2 [/mathjaxinline]', 'Mixed fractions: 1 1/2 = 3/2'), + ( + 'Converting decimals to mixed fractions: [mathjaxinline] 2.5 = 5/2 [/mathjaxinline]', + 'Converting decimals to mixed fractions: 2.5 = 5/2', + ), + ( + 'Trig identities: [mathjaxinline] \\sin(x + y) = \\sin(x) \\cos(y) + \\cos(x) \\sin(y) [/mathjaxinline]', + 'Trig identities: sin(x + y) = sin(x) cos(y) + cos(x) sin(y)', + ), + ( + 'Sine, cosine, and tangent: [mathjaxinline] \\sin(x) [/mathjaxinline] [mathjaxinline] \\cos(x) [/mathjaxinline] [mathjaxinline] \\tan(x) [/mathjaxinline]', + 'Sine, cosine, and tangent: sin(x) cos(x) tan(x)', + ), + ( + 'Hyperbolic trig functions: [mathjaxinline] \\sinh(x) [/mathjaxinline] [mathjaxinline] \\cosh(x) [/mathjaxinline]', + 'Hyperbolic trig functions: sinh(x) cosh(x)', + ), + ( + "Simple derivative: [mathjax] f(x) = x^2, f'(x) = 2x [/mathjax]", + "Simple derivative: f(x) = x², f'(x) = 2x", + ), + ('Double integral: [mathjax] int\\int (x + y) dxdy [/mathjax]', 'Double integral: int∫ (x + y) dxdy'), + ( + 'Partial derivatives: [mathjax] f(x,y) = xy, \\frac{\\partial f}{\\partial x} = y [/mathjax] [mathjax] \\frac{\\partial f}{\\partial y} = x [/mathjax]', + 'Partial derivatives: f(x,y) = xy, (∂ f/∂ x) = y (∂ f/∂ y) = x', + ), + ( + 'Mean and standard deviation: [mathjax] mu = 2, \\sigma = 1 [/mathjax]', + 'Mean and standard deviation: mu = 2, σ = 1', + ), + ( + 'Binomial probability: [mathjax] P(X = k) = (\\binom{n}{k} p^k (1-p)^{n-k}) [/mathjax]', + 'Binomial probability: P(X = k) = (\\binom{n}{k} pᵏ (1−p)ⁿ⁻ᵏ)', + ), + ('Gaussian distribution: [mathjax] N(\\mu, \\sigma^2) [/mathjax]', 'Gaussian distribution: N(μ, σ²)'), + ( + 'Greek letters: [mathjaxinline] \\alpha [/mathjaxinline] [mathjaxinline] \\beta [/mathjaxinline] [mathjaxinline] \\gamma [/mathjaxinline]', + 'Greek letters: α β γ', + ), + ( + 'Subscripted variables: [mathjaxinline] x_i [/mathjaxinline] [mathjaxinline] y_j [/mathjaxinline]', + 'Subscripted variables: xᵢ yⱼ', + ), + ('Superscripted variables: [mathjaxinline] x^{i} [/mathjaxinline]', 'Superscripted variables: xⁱ'), + ( + 'Not supported: \\( \\begin{bmatrix} 1 & 0 \\ 0 & 1 \\end{bmatrix} = I \\)', + 'Not supported: \\begin{bmatrix} 1 & 0 \\ 0 & 1 \\end{bmatrix} = I', + ), + ( + 'Bold text: \\( {\\bf a} \\cdot {\\bf b} = |{\\bf a}| |{\\bf b}| \\cos(\\theta) \\)', + 'Bold text: a ⋅ b = |a| |b| cos(θ)', + ), + ('Bold text: \\( \\frac{\\sqrt{\\mathbf{2}+3}}{\\sqrt{4}} \\)', 'Bold text: (√{2+3}/√{4})'), + ('Nested Bold text 1: \\( \\mathbf{ \\frac{1}{2} } \\)', 'Nested Bold text 1: (1/2)'), + ( + 'Nested Bold text 2: \\( \\mathbf{a \\cdot (a \\mathbf{\\times} b)} \\)', + 'Nested Bold text 2: a ⋅ (a × b)' + ), + ( + 'Nested Bold text 3: \\( \\mathbf{a \\cdot (a \\bm{\\times} b)} \\)', + 'Nested Bold text 3: a ⋅ (a × b)' + ), + ('Sqrt test 1: \\(\\sqrt\\)', 'Sqrt test 1: √'), + ('Sqrt test 2: \\(x^2 + \\sqrt(y)\\)', 'Sqrt test 2: x² + √(y)'), + ('Sqrt test 3: [mathjaxinline]x^2 + \\sqrt(y)[/mathjaxinline]', 'Sqrt test 3: x² + √(y)'), + ('Fraction test 1: \\( \\frac{2} {3} \\)', 'Fraction test 1: (2/3)'), + ('Fraction test 2: \\( \\frac{2}{3} \\)', 'Fraction test 2: (2/3)'), + ('Fraction test 3: \\( \\frac{\\frac{2}{3}}{4} \\)', 'Fraction test 3: ((2/3)/4)'), + ('Fraction test 4: \\( \\frac{\\frac{2} {3}}{4} \\)', 'Fraction test 4: ((2/3)/4)'), + ('Fraction test 5: \\( \\frac{\\frac{2} {3}}{\\frac{4}{3}} \\)', 'Fraction test 5: ((2/3)/(4/3))'), + # Invalid equations. + ('Fraction error: \\( \\frac{2} \\)', 'Fraction error: \\frac{2}'), + ('Fraction error 2: \\( \\frac{\\frac{2}{3}{4} \\)', 'Fraction error 2: \\frac{\\frac{2}{3}{4}'), + ('Unclosed: [mathjaxinline]x^2', 'Unclosed: [mathjaxinline]x^2'), + ( + 'Missing closing bracket: \\( \\frac{\\frac{2} {3}{\\frac{4}{3}} \\)', + 'Missing closing bracket: \\frac{\\frac{2} {3}{\\frac{4}{3}}' + ), + ('No equation: normal text', 'No equation: normal text'), + ] + # pylint: enable=line-too-long + block = BlockFactory.create( + parent_location=self.toy_course.location, + category="html", + display_name="Non-default HTML Block", + editor="raw", + use_latex_compiler=True, + data="|||".join(e[0] for e in eqns), + ) + doc = {} + doc.update(searchable_doc_for_course_block(block)) + doc.update(searchable_doc_tags(block.usage_key)) + result = doc['description'].split('|||') + for i, eqn in enumerate(result): + assert eqn.strip() == eqns[i][1] diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index d73ac89b2a0a..a6f79ba3cd7d 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -1208,6 +1208,8 @@ unicodecsv==0.14.1 # via # -r requirements/edx/kernel.in # edx-enterprise +unicodeit==0.7.5 + # via -r requirements/edx/kernel.in uritemplate==4.1.1 # via # drf-spectacular diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index a153f18be7a3..e353c37e9541 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -2160,6 +2160,10 @@ unicodecsv==0.14.1 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-enterprise +unicodeit==0.7.5 + # via + # -r requirements/edx/doc.txt + # -r requirements/edx/testing.txt unidiff==0.7.5 # via -r requirements/edx/testing.txt uritemplate==4.1.1 diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index f715e876d7f6..68cebb2d3e6d 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -1521,6 +1521,8 @@ unicodecsv==0.14.1 # via # -r requirements/edx/base.txt # edx-enterprise +unicodeit==0.7.5 + # via -r requirements/edx/base.txt uritemplate==4.1.1 # via # -r requirements/edx/base.txt diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index d2ec04314801..d1a132778133 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -163,3 +163,4 @@ web-fragments # Provides the ability to render fragments o wrapt # Better functools.wrapped. TODO: functools has since improved, maybe we can switch? XBlock[django] # Courseware component architecture xss-utils # https://github.com/openedx/edx-platform/pull/20633 Fix XSS via Translations +unicodeit # Converts mathjax equation to plain text by using unicode symbols diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index f6ad62bf0d5d..e27fb1195b95 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -1605,6 +1605,8 @@ unicodecsv==0.14.1 # via # -r requirements/edx/base.txt # edx-enterprise +unicodeit==0.7.5 + # via -r requirements/edx/base.txt unidiff==0.7.5 # via -r requirements/edx/testing.in uritemplate==4.1.1 From bdef5ad6aa6254d873d470e22c19312d55c4bf30 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 13 Jan 2025 14:41:30 -0500 Subject: [PATCH 03/18] test: Drop some stub config. I believe this was only used with BokChoy and just got missed in the various cleanup. Removing it now since we're not using it. --- common/djangoapps/terrain/__init__.py | 0 common/djangoapps/terrain/stubs/__init__.py | 0 common/djangoapps/terrain/stubs/catalog.py | 60 --- common/djangoapps/terrain/stubs/comments.py | 145 ------- .../terrain/stubs/data/ora_graded_rubric.xml | 1 - .../terrain/stubs/data/ora_rubric.xml | 1 - common/djangoapps/terrain/stubs/ecommerce.py | 64 --- common/djangoapps/terrain/stubs/edxnotes.py | 395 ------------------ common/djangoapps/terrain/stubs/http.py | 281 ------------- common/djangoapps/terrain/stubs/lti.py | 317 -------------- common/djangoapps/terrain/stubs/start.py | 109 ----- .../terrain/stubs/tests/__init__.py | 0 .../terrain/stubs/tests/test_edxnotes.py | 372 ----------------- .../terrain/stubs/tests/test_http.py | 124 ------ .../terrain/stubs/tests/test_lti_stub.py | 98 ----- .../terrain/stubs/tests/test_video.py | 48 --- .../terrain/stubs/tests/test_xqueue_stub.py | 173 -------- .../terrain/stubs/tests/test_youtube_stub.py | 71 ---- .../djangoapps/terrain/stubs/video_source.py | 60 --- common/djangoapps/terrain/stubs/xqueue.py | 226 ---------- common/djangoapps/terrain/stubs/youtube.py | 172 -------- conf/locale/config.yaml | 1 - scripts/vulture/find-dead-code.sh | 4 +- 23 files changed, 2 insertions(+), 2720 deletions(-) delete mode 100644 common/djangoapps/terrain/__init__.py delete mode 100644 common/djangoapps/terrain/stubs/__init__.py delete mode 100644 common/djangoapps/terrain/stubs/catalog.py delete mode 100644 common/djangoapps/terrain/stubs/comments.py delete mode 100644 common/djangoapps/terrain/stubs/data/ora_graded_rubric.xml delete mode 100644 common/djangoapps/terrain/stubs/data/ora_rubric.xml delete mode 100644 common/djangoapps/terrain/stubs/ecommerce.py delete mode 100644 common/djangoapps/terrain/stubs/edxnotes.py delete mode 100644 common/djangoapps/terrain/stubs/http.py delete mode 100644 common/djangoapps/terrain/stubs/lti.py delete mode 100644 common/djangoapps/terrain/stubs/start.py delete mode 100644 common/djangoapps/terrain/stubs/tests/__init__.py delete mode 100644 common/djangoapps/terrain/stubs/tests/test_edxnotes.py delete mode 100644 common/djangoapps/terrain/stubs/tests/test_http.py delete mode 100644 common/djangoapps/terrain/stubs/tests/test_lti_stub.py delete mode 100644 common/djangoapps/terrain/stubs/tests/test_video.py delete mode 100644 common/djangoapps/terrain/stubs/tests/test_xqueue_stub.py delete mode 100644 common/djangoapps/terrain/stubs/tests/test_youtube_stub.py delete mode 100644 common/djangoapps/terrain/stubs/video_source.py delete mode 100644 common/djangoapps/terrain/stubs/xqueue.py delete mode 100644 common/djangoapps/terrain/stubs/youtube.py diff --git a/common/djangoapps/terrain/__init__.py b/common/djangoapps/terrain/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/common/djangoapps/terrain/stubs/__init__.py b/common/djangoapps/terrain/stubs/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/common/djangoapps/terrain/stubs/catalog.py b/common/djangoapps/terrain/stubs/catalog.py deleted file mode 100644 index 1767485028a6..000000000000 --- a/common/djangoapps/terrain/stubs/catalog.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Stub implementation of catalog service for acceptance tests -""" -# pylint: disable=invalid-name - - -import re - -import six.moves.urllib.parse - -from .http import StubHttpRequestHandler, StubHttpService - - -class StubCatalogServiceHandler(StubHttpRequestHandler): # lint-amnesty, pylint: disable=missing-class-docstring - - def do_GET(self): # lint-amnesty, pylint: disable=missing-function-docstring - pattern_handlers = { - r'/api/v1/programs/$': self.program_list, - r'/api/v1/programs/([0-9a-f-]+)/$': self.program_detail, - r'/api/v1/program_types/$': self.program_types, - r'/api/v1/pathways/$': self.pathways - } - - if self.match_pattern(pattern_handlers): - return - - self.send_response(404, content='404 Not Found') - - def match_pattern(self, pattern_handlers): - """ - Find the correct handler method given the path info from the HTTP request. - """ - path = six.moves.urllib.parse.urlparse(self.path).path - for pattern, handler in pattern_handlers.items(): - match = re.match(pattern, path) - if match: - handler(*match.groups()) - return True - - def program_list(self): - """Stub the catalog's program list endpoint.""" - programs = self.server.config.get('catalog.programs', []) - self.send_json_response(programs) - - def program_detail(self, program_uuid): - """Stub the catalog's program detail endpoint.""" - program = self.server.config.get('catalog.programs.' + program_uuid) - self.send_json_response(program) - - def program_types(self): - program_types = self.server.config.get('catalog.programs_types', []) - self.send_json_response(program_types) - - def pathways(self): - pathways = self.server.config.get('catalog.pathways', []) - self.send_json_response(pathways) - - -class StubCatalogService(StubHttpService): - HANDLER_CLASS = StubCatalogServiceHandler diff --git a/common/djangoapps/terrain/stubs/comments.py b/common/djangoapps/terrain/stubs/comments.py deleted file mode 100644 index 8fef4b33e361..000000000000 --- a/common/djangoapps/terrain/stubs/comments.py +++ /dev/null @@ -1,145 +0,0 @@ -""" -Stub implementation of cs_comments_service for acceptance tests -""" - - -import re -from collections import OrderedDict - -import six.moves.urllib.parse - -from .http import StubHttpRequestHandler, StubHttpService - - -class StubCommentsServiceHandler(StubHttpRequestHandler): # lint-amnesty, pylint: disable=missing-class-docstring - - @property - def _params(self): - return six.moves.urllib.parse.parse_qs(six.moves.urllib.parse.urlparse(self.path).query) - - def do_GET(self): # lint-amnesty, pylint: disable=missing-function-docstring - pattern_handlers = OrderedDict([ - ("/api/v1/users/(?P\\d+)/active_threads$", self.do_user_profile), - ("/api/v1/users/(?P\\d+)$", self.do_user), - ("/api/v1/search/threads$", self.do_search_threads), - ("/api/v1/threads$", self.do_threads), - ("/api/v1/threads/(?P\\w+)$", self.do_thread), - ("/api/v1/comments/(?P\\w+)$", self.do_comment), - ("/api/v1/(?P\\w+)/threads$", self.do_commentable), - ]) - if self.match_pattern(pattern_handlers): - return - - self.send_response(404, content="404 Not Found") - - def match_pattern(self, pattern_handlers): # lint-amnesty, pylint: disable=missing-function-docstring - path = six.moves.urllib.parse.urlparse(self.path).path - for pattern in pattern_handlers: - match = re.match(pattern, path) - if match: - pattern_handlers[pattern](**match.groupdict()) - return True - return None - - def do_PUT(self): - if self.path.startswith('/set_config'): - return StubHttpRequestHandler.do_PUT(self) - pattern_handlers = { - "/api/v1/users/(?P\\d+)$": self.do_put_user, - } - if self.match_pattern(pattern_handlers): - return - self.send_response(204, "") - - def do_put_user(self, user_id): # lint-amnesty, pylint: disable=unused-argument - self.server.config['default_sort_key'] = self.post_dict.get("default_sort_key", "date") - self.send_json_response({'username': self.post_dict.get("username"), 'external_id': self.post_dict.get("external_id")}) # lint-amnesty, pylint: disable=line-too-long - - def do_DELETE(self): # lint-amnesty, pylint: disable=missing-function-docstring - pattern_handlers = { - "/api/v1/comments/(?P\\w+)$": self.do_delete_comment - } - if self.match_pattern(pattern_handlers): - return - self.send_json_response({}) - - def do_user(self, user_id): # lint-amnesty, pylint: disable=missing-function-docstring - response = { - "id": user_id, - "default_sort_key": self.server.config.get("default_sort_key", "date"), - "upvoted_ids": [], - "downvoted_ids": [], - "subscribed_thread_ids": [], - } - if 'course_id' in self._params: - response.update({ - "threads_count": 1, - "comments_count": 2 - }) - self.send_json_response(response) - - def do_user_profile(self, user_id): # lint-amnesty, pylint: disable=missing-function-docstring, unused-argument - if 'active_threads' in self.server.config: - user_threads = self.server.config['active_threads'][:] - params = self._params - page = int(params.get("page", ["1"])[0]) - per_page = int(params.get("per_page", ["20"])[0]) - num_pages = max(len(user_threads) - 1, 1) / per_page + 1 - user_threads = user_threads[(page - 1) * per_page:page * per_page] - self.send_json_response({ - "collection": user_threads, - "page": page, - "num_pages": num_pages - }) - else: - self.send_response(404, content="404 Not Found") - - def do_thread(self, thread_id): # lint-amnesty, pylint: disable=missing-function-docstring - if thread_id in self.server.config.get('threads', {}): - thread = self.server.config['threads'][thread_id].copy() - params = six.moves.urllib.parse.parse_qs(six.moves.urllib.parse.urlparse(self.path).query) - if "recursive" in params and params["recursive"][0] == "True": - thread.setdefault('children', []) - resp_total = thread.setdefault('resp_total', len(thread['children'])) # lint-amnesty, pylint: disable=unused-variable - resp_skip = int(params.get("resp_skip", ["0"])[0]) - resp_limit = int(params.get("resp_limit", ["10000"])[0]) - thread['children'] = thread['children'][resp_skip:(resp_skip + resp_limit)] - self.send_json_response(thread) - else: - self.send_response(404, content="404 Not Found") - - def do_threads(self): - threads = self.server.config.get('threads', {}) - threads_data = list(threads.values()) - self.send_json_response({"collection": threads_data, "page": 1, "num_pages": 1}) - - def do_search_threads(self): - self.send_json_response(self.server.config.get('search_result', {})) - - def do_comment(self, comment_id): - # django_comment_client calls GET comment before doing a DELETE, so that's what this is here to support. - if comment_id in self.server.config.get('comments', {}): - comment = self.server.config['comments'][comment_id] - self.send_json_response(comment) - - def do_delete_comment(self, comment_id): - """Handle comment deletion. Returns a JSON representation of the - deleted comment.""" - if comment_id in self.server.config.get('comments', {}): - comment = self.server.config['comments'][comment_id] - self.send_json_response(comment) - - def do_commentable(self, commentable_id): - self.send_json_response({ - "collection": [ - thread - for thread in self.server.config.get('threads', {}).values() - if thread.get('commentable_id') == commentable_id - ], - "page": 1, - "num_pages": 1, - }) - - -class StubCommentsService(StubHttpService): - HANDLER_CLASS = StubCommentsServiceHandler diff --git a/common/djangoapps/terrain/stubs/data/ora_graded_rubric.xml b/common/djangoapps/terrain/stubs/data/ora_graded_rubric.xml deleted file mode 100644 index 5db0138ebe34..000000000000 --- a/common/djangoapps/terrain/stubs/data/ora_graded_rubric.xml +++ /dev/null @@ -1 +0,0 @@ -Writing Applications0 Language Conventions 1 diff --git a/common/djangoapps/terrain/stubs/data/ora_rubric.xml b/common/djangoapps/terrain/stubs/data/ora_rubric.xml deleted file mode 100644 index 14959de008e0..000000000000 --- a/common/djangoapps/terrain/stubs/data/ora_rubric.xml +++ /dev/null @@ -1 +0,0 @@ -Writing Applications Language Conventions diff --git a/common/djangoapps/terrain/stubs/ecommerce.py b/common/djangoapps/terrain/stubs/ecommerce.py deleted file mode 100644 index 96835ab0c171..000000000000 --- a/common/djangoapps/terrain/stubs/ecommerce.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Stub implementation of ecommerce service for acceptance tests -""" - - -import re - -import six.moves.urllib.parse - -from .http import StubHttpRequestHandler, StubHttpService - - -class StubEcommerceServiceHandler(StubHttpRequestHandler): # pylint: disable=missing-class-docstring - - # pylint: disable=missing-function-docstring - def do_GET(self): - pattern_handlers = { - '/api/v2/orders/$': self.get_orders_list, - } - if self.match_pattern(pattern_handlers): - return - self.send_response(404, content='404 Not Found') - - def match_pattern(self, pattern_handlers): - """ - Find the correct handler method given the path info from the HTTP request. - """ - path = six.moves.urllib.parse.urlparse(self.path).path - for pattern in pattern_handlers: - match = re.match(pattern, path) - if match: - pattern_handlers[pattern](**match.groupdict()) - return True - return None - - def get_orders_list(self): - """ - Stubs the orders list endpoint. - """ - orders = { - 'results': [ - { - 'status': 'Complete', - 'number': 'Edx-123', - 'total_excl_tax': '100.00', - 'date_placed': '2016-04-21T23:14:23Z', - 'lines': [ - { - 'title': 'Test Course', - 'line_price_excl_tax': '100.00', - 'product': { - 'product_class': 'Seat' - } - } - ], - } - ] - } - orders = self.server.config.get('orders', orders) - self.send_json_response(orders) - - -class StubEcommerceService(StubHttpService): - HANDLER_CLASS = StubEcommerceServiceHandler diff --git a/common/djangoapps/terrain/stubs/edxnotes.py b/common/djangoapps/terrain/stubs/edxnotes.py deleted file mode 100644 index a147825c25d9..000000000000 --- a/common/djangoapps/terrain/stubs/edxnotes.py +++ /dev/null @@ -1,395 +0,0 @@ -""" -Stub implementation of EdxNotes for acceptance tests -""" - - -import json -import re -from copy import deepcopy -from datetime import datetime -from math import ceil -from uuid import uuid4 - -from urllib.parse import urlencode - -from .http import StubHttpRequestHandler, StubHttpService - - -class StubEdxNotesServiceHandler(StubHttpRequestHandler): - """ - Handler for EdxNotes requests. - """ - URL_HANDLERS = { - "GET": { - "/api/v1/annotations$": "_collection", - "/api/v1/annotations/(?P[0-9A-Fa-f]+)$": "_read", - "/api/v1/search$": "_search", - }, - "POST": { - "/api/v1/annotations$": "_create", - "/create_notes": "_create_notes", - }, - "PUT": { - "/api/v1/annotations/(?P[0-9A-Fa-f]+)$": "_update", - "/cleanup$": "_cleanup", - }, - "DELETE": { - "/api/v1/annotations/(?P[0-9A-Fa-f]+)$": "_delete", - }, - } - - def _match_pattern(self, pattern_handlers): - """ - Finds handler by the provided handler patterns and delegate response to - the matched handler. - """ - for pattern in pattern_handlers: - match = re.match(pattern, self.path_only) - if match: - handler = getattr(self, pattern_handlers[pattern], None) - if handler: - handler(**match.groupdict()) - return True - return None - - def _send_handler_response(self, method): - """ - Delegate response to handler methods. - If no handler defined, send a 404 response. - """ - # Choose the list of handlers based on the HTTP method - if method in self.URL_HANDLERS: - handlers_list = self.URL_HANDLERS[method] - else: - self.log_error(f"Unrecognized method '{method}'") - return - - # Check the path (without querystring params) against our list of handlers - if self._match_pattern(handlers_list): - return - # If we don't have a handler for this URL and/or HTTP method, - # respond with a 404. - else: - self.send_response(404, content="404 Not Found") - - def do_GET(self): - """ - Handle GET methods to the EdxNotes API stub. - """ - self._send_handler_response("GET") - - def do_POST(self): - """ - Handle POST methods to the EdxNotes API stub. - """ - self._send_handler_response("POST") - - def do_PUT(self): - """ - Handle PUT methods to the EdxNotes API stub. - """ - if self.path.startswith("/set_config"): - return StubHttpRequestHandler.do_PUT(self) - - self._send_handler_response("PUT") - - def do_DELETE(self): - """ - Handle DELETE methods to the EdxNotes API stub. - """ - self._send_handler_response("DELETE") - - def do_OPTIONS(self): - """ - Handle OPTIONS methods to the EdxNotes API stub. - """ - self.send_response(200, headers={ - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS", - "Access-Control-Allow-Headers": "Content-Length, Content-Type, X-Annotator-Auth-Token, X-Requested-With, X-Annotator-Auth-Token, X-Requested-With, X-CSRFToken", # lint-amnesty, pylint: disable=line-too-long - }) - - def respond(self, status_code=200, content=None): - """ - Send a response back to the client with the HTTP `status_code` (int), - the given content serialized as JSON (str), and the headers set appropriately. - """ - headers = { - "Access-Control-Allow-Origin": "*", - } - if status_code < 400 and content: - headers["Content-Type"] = "application/json" - content = json.dumps(content).encode('utf-8') - else: - headers["Content-Type"] = "text/html" - - self.send_response(status_code, content, headers) - - def _create(self): - """ - Create a note, assign id, annotator_schema_version, created and updated dates. - """ - note = json.loads(self.request_content.decode('utf-8')) - note.update({ - "id": uuid4().hex, - "annotator_schema_version": "v1.0", - "created": datetime.utcnow().isoformat(), - "updated": datetime.utcnow().isoformat(), - }) - self.server.add_notes(note) - self.respond(content=note) - - def _create_notes(self): - """ - The same as self._create, but it works a list of notes. - """ - try: - notes = json.loads(self.request_content.decode('utf-8')) - except ValueError: - self.respond(400, "Bad Request") - return - - if not isinstance(notes, list): - self.respond(400, "Bad Request") - return - - for note in notes: - note.update({ - "id": uuid4().hex, - "annotator_schema_version": "v1.0", - "created": note["created"] if note.get("created") else datetime.utcnow().isoformat(), - "updated": note["updated"] if note.get("updated") else datetime.utcnow().isoformat(), - }) - self.server.add_notes(note) - - self.respond(content=notes) - - def _read(self, note_id): - """ - Return the note by note id. - """ - notes = self.server.get_all_notes() - result = self.server.filter_by_id(notes, note_id) - if result: - self.respond(content=result[0]) - else: - self.respond(404, "404 Not Found") - - def _update(self, note_id): - """ - Update the note by note id. - """ - note = self.server.update_note(note_id, json.loads(self.request_content.decode('utf-8'))) - if note: - self.respond(content=note) - else: - self.respond(404, "404 Not Found") - - def _delete(self, note_id): - """ - Delete the note by note id. - """ - if self.server.delete_note(note_id): - self.respond(204, "No Content") - else: - self.respond(404, "404 Not Found") - - @staticmethod - def _get_next_prev_url(url_path, query_params, page_num, page_size): - """ - makes url with the query params including pagination params - for pagination next and previous urls - """ - query_params = deepcopy(query_params) - query_params.update({ - "page": page_num, - "page_size": page_size - }) - return url_path + "?" + urlencode(query_params) - - def _get_paginated_response(self, notes, page_num, page_size): - """ - Returns a paginated response of notes. - """ - start = (page_num - 1) * page_size - end = start + page_size - total_notes = len(notes) - url_path = "http://{server_address}:{port}{path}".format( - server_address=self.client_address[0], - port=self.server.port, - path=self.path_only - ) - - next_url = None if end >= total_notes else self._get_next_prev_url( - url_path, self.get_params, page_num + 1, page_size - ) - prev_url = None if page_num == 1 else self._get_next_prev_url( - url_path, self.get_params, page_num - 1, page_size) - - # Get notes from range - notes = deepcopy(notes[start:end]) - - paginated_response = { - 'total': total_notes, - 'num_pages': int(ceil(float(total_notes) / page_size)), - 'current_page': page_num, - 'rows': notes, - 'next': next_url, - 'start': start, - 'previous': prev_url - } - - return paginated_response - - def _search(self): - """ - Search for a notes by user id, course_id and usage_id. - """ - search_with_usage_id = False - user = self.get_params.get("user", None) - usage_ids = self.get_params.get("usage_id", []) - course_id = self.get_params.get("course_id", None) - text = self.get_params.get("text", None) - page = int(self.get_params.get("page", 1)) - page_size = int(self.get_params.get("page_size", 2)) - - if user is None: - self.respond(400, "Bad Request") - return - - notes = self.server.get_all_notes() - if course_id is not None: - notes = self.server.filter_by_course_id(notes, course_id) - if len(usage_ids) > 0: - search_with_usage_id = True - notes = self.server.filter_by_usage_id(notes, usage_ids) - if text: - notes = self.server.search(notes, text) - if not search_with_usage_id: - notes = self._get_paginated_response(notes, page, page_size) - self.respond(content=notes) - - def _collection(self): - """ - Return all notes for the user. - """ - user = self.get_params.get("user", None) - page = int(self.get_params.get("page", 1)) - page_size = int(self.get_params.get("page_size", 2)) - notes = self.server.get_all_notes() - - if user is None: - self.send_response(400, content="Bad Request") - return - notes = self._get_paginated_response(notes, page, page_size) - self.respond(content=notes) - - def _cleanup(self): - """ - Helper method that removes all notes to the stub EdxNotes service. - """ - self.server.cleanup() - self.respond() - - -class StubEdxNotesService(StubHttpService): - """ - Stub EdxNotes service. - """ - HANDLER_CLASS = StubEdxNotesServiceHandler - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.notes = [] - - def get_all_notes(self): - """ - Returns a list of all notes without pagination - """ - notes = deepcopy(self.notes) - notes.reverse() - return notes - - def add_notes(self, notes): - """ - Adds `notes(list)` to the stub EdxNotes service. - """ - if not isinstance(notes, list): - notes = [notes] - - for note in notes: - self.notes.append(note) - - def update_note(self, note_id, note_info): - """ - Updates the note with `note_id(str)` by the `note_info(dict)` to the - stub EdxNotes service. - """ - note = self.filter_by_id(self.notes, note_id) - if note: - note[0].update(note_info) - return note - else: - return None - - def delete_note(self, note_id): - """ - Removes the note with `note_id(str)` to the stub EdxNotes service. - """ - note = self.filter_by_id(self.notes, note_id) - if note: - index = self.notes.index(note[0]) - self.notes.pop(index) - return True - else: - return False - - def cleanup(self): - """ - Removes all notes to the stub EdxNotes service. - """ - self.notes = [] - - def filter_by_id(self, data, note_id): - """ - Filters provided `data(list)` by the `note_id(str)`. - """ - return self.filter_by(data, "id", note_id) - - def filter_by_user(self, data, user): - """ - Filters provided `data(list)` by the `user(str)`. - """ - return self.filter_by(data, "user", user) - - def filter_by_usage_id(self, data, usage_ids): - """ - Filters provided `data(list)` by the `usage_id(str)`. - """ - if not isinstance(usage_ids, list): - usage_ids = [usage_ids] - return self.filter_by_list(data, "usage_id", usage_ids) - - def filter_by_course_id(self, data, course_id): - """ - Filters provided `data(list)` by the `course_id(str)`. - """ - return self.filter_by(data, "course_id", course_id) - - def filter_by(self, data, field_name, value): - """ - Filters provided `data(list)` by the `field_name(str)` with `value`. - """ - return [note for note in data if note.get(field_name) == value] - - def filter_by_list(self, data, field_name, values): - """ - Filters provided `data(list)` by the `field_name(str)` in values. - """ - return [note for note in data if note.get(field_name) in values] - - def search(self, data, query): - """ - Search the `query(str)` text in the provided `data(list)`. - """ - return [note for note in data if str(query).strip() in note.get("text", "").split()] diff --git a/common/djangoapps/terrain/stubs/http.py b/common/djangoapps/terrain/stubs/http.py deleted file mode 100644 index 9dae75e21570..000000000000 --- a/common/djangoapps/terrain/stubs/http.py +++ /dev/null @@ -1,281 +0,0 @@ -""" -Stub implementation of an HTTP service. -""" - - -import json -import threading -from functools import wraps -from logging import getLogger - -import six -from lazy import lazy -from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer -from six.moves.socketserver import ThreadingMixIn - -LOGGER = getLogger(__name__) - - -def require_params(method, *required_keys): - """ - Decorator to ensure that the method has all the required parameters. - - Example: - - @require_params('GET', 'id', 'state') - def handle_request(self): - # .... - - would send a 400 response if no GET parameters were specified - for 'id' or 'state' (or if those parameters had empty values). - - The wrapped function should be a method of a `StubHttpRequestHandler` - subclass. - - Currently, "GET" and "POST" are the only supported methods. - """ - def decorator(func): - @wraps(func) - def wrapper(self, *args, **kwargs): - - # Read either GET querystring params or POST dict params - if method == "GET": - params = self.get_params - elif method == "POST": - params = self.post_dict - else: - raise ValueError(f"Unsupported method '{method}'") - - # Check for required values - missing = [] - for key in required_keys: - if params.get(key) is None: - missing.append(key) - - if len(missing) > 0: - msg = "Missing required key(s) {keys}".format(keys=",".join(missing)) - self.send_response(400, content=msg, headers={'Content-type': 'text/plain'}) - - # If nothing is missing, execute the function as usual - else: - return func(self, *args, **kwargs) - return wrapper - return decorator - - -class StubHttpRequestHandler(BaseHTTPRequestHandler): - """ - Handler for the stub HTTP service. - """ - - protocol = "HTTP/1.0" - - def log_message(self, format_str, *args): - """ - Redirect messages to keep the test console clean. - """ - LOGGER.debug(self._format_msg(format_str, *args)) - - def log_error(self, format_str, *args): - """ - Helper to log a server error. - """ - LOGGER.error(self._format_msg(format_str, *args)) - - @lazy - def request_content(self): - """ - Retrieve the content of the request. - """ - try: - length = int(self.headers.get('content-length')) - - except (TypeError, ValueError): - return "" - else: - return self.rfile.read(length) - - @lazy - def post_dict(self): - """ - Retrieve the request POST parameters from the client as a dictionary. - If no POST parameters can be interpreted, return an empty dict. - """ - - if isinstance(self.request_content, bytes): - contents = self.request_content.decode('utf-8') - else: - contents = self.request_content - - # The POST dict will contain a list of values for each key. - # None of our parameters are lists, however, so we map [val] --> val - # If the list contains multiple entries, we pick the first one - try: - post_dict = six.moves.urllib.parse.parse_qs(contents, keep_blank_values=True) - return { - key: list_val[0] - for key, list_val in post_dict.items() - } - - except: # lint-amnesty, pylint: disable=bare-except - return {} - - @lazy - def get_params(self): - """ - Return the GET parameters (querystring in the URL). - """ - query = six.moves.urllib.parse.urlparse(self.path).query - - # By default, `parse_qs` returns a list of values for each param - # For convenience, we replace lists of 1 element with just the element - return { - key: value[0] if len(value) == 1 else value - for key, value in six.moves.urllib.parse.parse_qs(query).items() - } - - @lazy - def path_only(self): - """ - Return the URL path without GET parameters. - Removes the trailing slash if there is one. - """ - path = six.moves.urllib.parse.urlparse(self.path).path - if path.endswith('/'): - return path[:-1] - else: - return path - - def do_PUT(self): - """ - Allow callers to configure the stub server using the /set_config URL. - The request should have POST data, such that: - - Each POST parameter is the configuration key. - Each POST value is a JSON-encoded string value for the configuration. - """ - if self.path in ("/set_config", "/set_config/"): - - if len(self.post_dict) > 0: - for key, value in self.post_dict.items(): - - self.log_message(f"Set config '{key}' to '{value}'") - - try: - value = json.loads(value) - - except ValueError: - self.log_message(f"Could not parse JSON: {value}") - self.send_response(400) - - else: - self.server.config[key] = value - self.send_response(200) - - # No parameters sent to configure, so return success by default - else: - self.send_response(200) - - else: - self.send_response(404) - - def send_response(self, status_code, content=None, headers=None): - """ - Send a response back to the client with the HTTP `status_code` (int), - `content` (str) and `headers` (dict). - """ - self.log_message( - f"Sent HTTP response: {status_code} with content '{content}' and headers {headers}" - ) - - if headers is None: - headers = { - 'Access-Control-Allow-Origin': "*", - } - - BaseHTTPRequestHandler.send_response(self, status_code) - - for (key, value) in headers.items(): - self.send_header(key, value) - - if len(headers) > 0: - self.end_headers() - - if content is not None: - if isinstance(content, str): - content = content.encode('utf-8') - self.wfile.write(content) - - def send_json_response(self, content): - """ - Send a response with status code 200, the given content serialized as - JSON, and the Content-Type header set appropriately - """ - self.send_response(200, json.dumps(content), {"Content-Type": "application/json"}) - - def _format_msg(self, format_str, *args): - """ - Format message for logging. - `format_str` is a string with old-style Python format escaping; - `args` is an array of values to fill into the string. - """ - if not args: - format_str = six.moves.urllib.parse.unquote(format_str) - return "{} - - [{}] {}\n".format( - self.client_address[0], - self.log_date_time_string(), - format_str % args - ) - - def do_HEAD(self): - """ - Respond to an HTTP HEAD request - """ - self.send_response(200) - - -class StubHttpService(ThreadingMixIn, HTTPServer): - """ - Stub HTTP service implementation. - """ - - # Subclasses override this to provide the handler class to use. - # Should be a subclass of `StubHttpRequestHandler` - HANDLER_CLASS = StubHttpRequestHandler - - def __init__(self, port_num=0): - """ - Configure the server to listen on localhost. - Default is to choose an arbitrary open port. - """ - address = ('0.0.0.0', port_num) - HTTPServer.__init__(self, address, self.HANDLER_CLASS) - - # Create a dict to store configuration values set by the client - self.config = {} - - # Start the server in a separate thread - server_thread = threading.Thread(target=self.serve_forever) - server_thread.daemon = True - server_thread.start() - - # Log the port we're using to help identify port conflict errors - LOGGER.debug(f'Starting service on port {self.port}') - - def shutdown(self): - """ - Stop the server and free up the port - """ - # First call superclass shutdown() - HTTPServer.shutdown(self) - - # We also need to manually close the socket - self.socket.close() - - @property - def port(self): - """ - Return the port that the service is listening on. - """ - _, port = self.server_address - return port diff --git a/common/djangoapps/terrain/stubs/lti.py b/common/djangoapps/terrain/stubs/lti.py deleted file mode 100644 index 46535abb9f80..000000000000 --- a/common/djangoapps/terrain/stubs/lti.py +++ /dev/null @@ -1,317 +0,0 @@ -""" -Stub implementation of LTI Provider. - -What is supported: ------------------- - -1.) This LTI Provider can service only one Tool Consumer at the same time. It is -not possible to have this LTI multiple times on a single page in LMS. - -""" - - -import base64 -import hashlib -import logging -import textwrap -from unittest import mock -from uuid import uuid4 - -import oauthlib.oauth1 -import requests -import six -from oauthlib.oauth1.rfc5849 import parameters, signature - -from openedx.core.djangolib.markup import HTML - -from .http import StubHttpRequestHandler, StubHttpService - -log = logging.getLogger(__name__) - - -class StubLtiHandler(StubHttpRequestHandler): - """ - A handler for LTI POST and GET requests. - """ - DEFAULT_CLIENT_KEY = 'test_client_key' - DEFAULT_CLIENT_SECRET = 'test_client_secret' - DEFAULT_LTI_ENDPOINT = 'correct_lti_endpoint' - DEFAULT_LTI_ADDRESS = 'http://{host}:{port}/' - - def do_GET(self): - """ - Handle a GET request from the client and sends response back. - - Used for checking LTI Provider started correctly. - """ - self.send_response(200, 'This is LTI Provider.', {'Content-type': 'text/plain'}) - - def do_POST(self): - """ - Handle a POST request from the client and sends response back. - """ - if 'grade' in self.path and self._send_graded_result().status_code == 200: - status_message = HTML('LTI consumer (edX) responded with XML content:
{grade_data}').format( - grade_data=self.server.grade_data['TC answer'] - ) - content = self._create_content(status_message) - self.send_response(200, content) - elif 'lti2_outcome' in self.path and self._send_lti2_outcome().status_code == 200: - status_message = HTML('LTI consumer (edX) responded with HTTP {}
').format( - self.server.grade_data['status_code']) - content = self._create_content(status_message) - self.send_response(200, content) - elif 'lti2_delete' in self.path and self._send_lti2_delete().status_code == 200: - status_message = HTML('LTI consumer (edX) responded with HTTP {}
').format( - self.server.grade_data['status_code']) - content = self._create_content(status_message) - self.send_response(200, content) - # Respond to request with correct lti endpoint - elif self._is_correct_lti_request(): - params = {k: v for k, v in self.post_dict.items() if k != 'oauth_signature'} - if self._check_oauth_signature(params, self.post_dict.get('oauth_signature', "")): - status_message = "This is LTI tool. Success." - # Set data for grades what need to be stored as server data - if 'lis_outcome_service_url' in self.post_dict: - self.server.grade_data = { - 'callback_url': self.post_dict.get('lis_outcome_service_url').replace('https', 'http'), - 'sourcedId': self.post_dict.get('lis_result_sourcedid') - } - host = self.server.server_address[0] - submit_url = f'//{host}:{self.server.server_address[1]}' - content = self._create_content(status_message, submit_url) - self.send_response(200, content) - else: - content = self._create_content("Wrong LTI signature") - self.send_response(200, content) - else: - content = self._create_content("Invalid request URL") - self.send_response(500, content) - - def _send_graded_result(self): - """ - Send grade request. - """ - values = { - 'textString': 0.5, - 'sourcedId': self.server.grade_data['sourcedId'], - 'imsx_messageIdentifier': uuid4().hex, - } - payload = textwrap.dedent(""" - - - - - V1.0 - {imsx_messageIdentifier} / - - - - - - - {sourcedId} - - - - en-us - {textString} - - - - - - - """) - - data = payload.format(**values) - url = self.server.grade_data['callback_url'] - headers = { - 'Content-Type': 'application/xml', - 'X-Requested-With': 'XMLHttpRequest', - 'Authorization': self._oauth_sign(url, data) - } - - # Send request ignoring verifirecation of SSL certificate - response = requests.post(url, data=data, headers=headers, verify=False) - - self.server.grade_data['TC answer'] = response.content - return response - - def _send_lti2_outcome(self): - """ - Send a grade back to consumer - """ - payload = textwrap.dedent(""" - {{ - "@context" : "http://purl.imsglobal.org/ctx/lis/v2/Result", - "@type" : "Result", - "resultScore" : {score}, - "comment" : "This is awesome." - }} - """) - data = payload.format(score=0.8) - return self._send_lti2(data) - - def _send_lti2_delete(self): - """ - Send a delete back to consumer - """ - payload = textwrap.dedent(""" - { - "@context" : "http://purl.imsglobal.org/ctx/lis/v2/Result", - "@type" : "Result" - } - """) - return self._send_lti2(payload) - - def _send_lti2(self, payload): - """ - Send lti2 json result service request. - """ - ### We compute the LTI V2.0 service endpoint from the callback_url (which is set by the launch call) - url = self.server.grade_data['callback_url'] - url_parts = url.split('/') - url_parts[-1] = "lti_2_0_result_rest_handler" - anon_id = self.server.grade_data['sourcedId'].split(":")[-1] - url_parts.extend(["user", anon_id]) - new_url = '/'.join(url_parts) - - content_type = 'application/vnd.ims.lis.v2.result+json' - headers = { - 'Content-Type': content_type, - 'Authorization': self._oauth_sign(new_url, payload, - method='PUT', - content_type=content_type) - } - - # Send request ignoring verifirecation of SSL certificate - response = requests.put(new_url, data=payload, headers=headers, verify=False) - self.server.grade_data['status_code'] = response.status_code - self.server.grade_data['TC answer'] = response.content - return response - - def _create_content(self, response_text, submit_url=None): - """ - Return content (str) either for launch, send grade or get result from TC. - """ - if submit_url: - submit_form = textwrap.dedent(HTML(""" -
- -
-
- -
-
- -
- """)).format(submit_url=submit_url) - else: - submit_form = '' - - # Show roles only for LTI launch. - if self.post_dict.get('roles'): - role = HTML('
Role: {}
').format(self.post_dict['roles']) - else: - role = '' - - response_str = textwrap.dedent(HTML(""" - - - TEST TITLE - - -
-

IFrame loaded

-

Server response is:

-

{response}

- {role} -
- {submit_form} - - - """)).format(response=response_text, role=role, submit_form=submit_form) - - # Currently LTI block doublequotes the lis_result_sourcedid parameter. - # Unquote response two times. - return six.moves.urllib.parse.unquote(six.moves.urllib.parse.unquote(response_str)) - - def _is_correct_lti_request(self): - """ - Return a boolean indicating whether the URL path is a valid LTI end-point. - """ - lti_endpoint = self.server.config.get('lti_endpoint', self.DEFAULT_LTI_ENDPOINT) - return lti_endpoint in self.path - - def _oauth_sign(self, url, body, content_type='application/x-www-form-urlencoded', method='POST'): - """ - Signs request and returns signed Authorization header. - """ - client_key = self.server.config.get('client_key', self.DEFAULT_CLIENT_KEY) - client_secret = self.server.config.get('client_secret', self.DEFAULT_CLIENT_SECRET) - client = oauthlib.oauth1.Client( - client_key=str(client_key), - client_secret=str(client_secret) - ) - headers = { - # This is needed for body encoding: - 'Content-Type': content_type, - } - - # Calculate and encode body hash. See http://oauth.googlecode.com/svn/spec/ext/body_hash/1.0/oauth-bodyhash.html - sha1 = hashlib.sha1() - sha1.update(body.encode('utf-8')) - oauth_body_hash = base64.b64encode(sha1.digest()).decode('utf-8') - mock_request = mock.Mock( - uri=str(six.moves.urllib.parse.unquote(url)), - headers=headers, - body="", - decoded_body="", - http_method=str(method), - ) - params = client.get_oauth_params(mock_request) - mock_request.oauth_params = params - mock_request.oauth_params.append(('oauth_body_hash', oauth_body_hash)) - sig = client.get_oauth_signature(mock_request) - mock_request.oauth_params.append(('oauth_signature', sig)) - new_headers = parameters.prepare_headers(mock_request.oauth_params, headers, realm=None) - return new_headers['Authorization'] - - def _check_oauth_signature(self, params, client_signature): - """ - Checks oauth signature from client. - - `params` are params from post request except signature, - `client_signature` is signature from request. - - Builds mocked request and verifies hmac-sha1 signing:: - 1. builds string to sign from `params`, `url` and `http_method`. - 2. signs it with `client_secret` which comes from server settings. - 3. obtains signature after sign and then compares it with request.signature - (request signature comes form client in request) - - Returns `True` if signatures are correct, otherwise `False`. - - """ - client_secret = str(self.server.config.get('client_secret', self.DEFAULT_CLIENT_SECRET)) - host = '127.0.0.1' - port = self.server.server_address[1] - lti_base = self.DEFAULT_LTI_ADDRESS.format(host=host, port=port) - lti_endpoint = self.server.config.get('lti_endpoint', self.DEFAULT_LTI_ENDPOINT) - url = lti_base + lti_endpoint - request = mock.Mock() - request.params = [(str(k), str(v)) for k, v in params.items()] - request.uri = str(url) - request.http_method = 'POST' - request.signature = str(client_signature) - return signature.verify_hmac_sha1(request, client_secret) - - -class StubLtiService(StubHttpService): - """ - A stub LTI provider server that responds - to POST and GET requests to localhost. - """ - - HANDLER_CLASS = StubLtiHandler diff --git a/common/djangoapps/terrain/stubs/start.py b/common/djangoapps/terrain/stubs/start.py deleted file mode 100644 index 976fb058f97b..000000000000 --- a/common/djangoapps/terrain/stubs/start.py +++ /dev/null @@ -1,109 +0,0 @@ -""" -Command-line utility to start a stub service. -""" - - -import logging -import sys -import time - -from .catalog import StubCatalogService -from .comments import StubCommentsService -from .ecommerce import StubEcommerceService -from .edxnotes import StubEdxNotesService -from .lti import StubLtiService -from .video_source import VideoSourceHttpService -from .xqueue import StubXQueueService -from .youtube import StubYouTubeService - -USAGE = "USAGE: python -m stubs.start SERVICE_NAME PORT_NUM [CONFIG_KEY=CONFIG_VAL, ...]" - -SERVICES = { - 'xqueue': StubXQueueService, - 'youtube': StubYouTubeService, - 'comments': StubCommentsService, - 'lti': StubLtiService, - 'video': VideoSourceHttpService, - 'edxnotes': StubEdxNotesService, - 'ecommerce': StubEcommerceService, - 'catalog': StubCatalogService, -} - -# Log to stdout, including debug messages -logging.basicConfig(level=logging.DEBUG, format="%(levelname)s %(message)s") - - -def get_args(): - """ - Parse arguments, returning tuple of `(service_name, port_num, config_dict)`. - Exits with a message if arguments are invalid. - """ - if len(sys.argv) < 3: - print(USAGE) - sys.exit(1) - - service_name = sys.argv[1] - port_num = sys.argv[2] - config_dict = _parse_config_args(sys.argv[3:]) - - if service_name not in SERVICES: - print("Unrecognized service '{}'. Valid choices are: {}".format( - service_name, ", ".join(list(SERVICES.keys())))) - sys.exit(1) - - try: - port_num = int(port_num) - if port_num < 0: - raise ValueError - - except ValueError: - print(f"Port '{port_num}' must be a positive integer") - sys.exit(1) - - return service_name, port_num, config_dict - - -def _parse_config_args(args): - """ - Parse stub configuration arguments, which are strings of the form "KEY=VAL". - `args` is a list of arguments from the command line. - Any argument that does not match the "KEY=VAL" format will be logged and skipped. - - Returns a dictionary with the configuration keys and values. - """ - config_dict = {} - for config_str in args: - try: - components = config_str.split('=') - if len(components) >= 2: - config_dict[components[0]] = "=".join(components[1:]) - - except: # lint-amnesty, pylint: disable=bare-except - print(f"Warning: could not interpret config value '{config_str}'") - - return config_dict - - -def main(): - """ - Start a server; shut down on keyboard interrupt signal. - """ - service_name, port_num, config_dict = get_args() - print(f"Starting stub service '{service_name}' on port {port_num}...") - - server = SERVICES[service_name](port_num=port_num) - server.config.update(config_dict) - - try: - while True: - time.sleep(1) - - except KeyboardInterrupt: - print("Stopping stub service...") - - finally: - server.shutdown() - - -if __name__ == "__main__": - main() diff --git a/common/djangoapps/terrain/stubs/tests/__init__.py b/common/djangoapps/terrain/stubs/tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/common/djangoapps/terrain/stubs/tests/test_edxnotes.py b/common/djangoapps/terrain/stubs/tests/test_edxnotes.py deleted file mode 100644 index 5d22d6c551f2..000000000000 --- a/common/djangoapps/terrain/stubs/tests/test_edxnotes.py +++ /dev/null @@ -1,372 +0,0 @@ -""" -Unit tests for stub EdxNotes implementation. -""" - - -import json -import unittest -from uuid import uuid4 - -import ddt -import requests -import six - -from ..edxnotes import StubEdxNotesService - - -@ddt.ddt -class StubEdxNotesServiceTest(unittest.TestCase): - """ - Test cases for the stub EdxNotes service. - """ - def setUp(self): - """ - Start the stub server. - """ - super().setUp() - self.server = StubEdxNotesService() - dummy_notes = self._get_dummy_notes(count=5) - self.server.add_notes(dummy_notes) - self.addCleanup(self.server.shutdown) - - def _get_dummy_notes(self, count=1): - """ - Returns a list of dummy notes. - """ - return [self._get_dummy_note(i) for i in range(count)] - - def _get_dummy_note(self, uid=0): - """ - Returns a single dummy note. - """ - nid = uuid4().hex - return { - "id": nid, - "created": "2014-10-31T10:05:00.000000", - "updated": "2014-10-31T10:50:00.101010", - "user": "dummy-user-id", - "usage_id": "dummy-usage-id-" + str(uid), - "course_id": "dummy-course-id", - "text": "dummy note text " + nid, - "quote": "dummy note quote", - "ranges": [ - { - "start": "/p[1]", - "end": "/p[1]", - "startOffset": 0, - "endOffset": 10, - } - ], - } - - def test_note_create(self): - dummy_note = { - "user": "dummy-user-id", - "usage_id": "dummy-usage-id", - "course_id": "dummy-course-id", - "text": "dummy note text", - "quote": "dummy note quote", - "ranges": [ - { - "start": "/p[1]", - "end": "/p[1]", - "startOffset": 0, - "endOffset": 10, - } - ], - } - response = requests.post(self._get_url("api/v1/annotations"), data=json.dumps(dummy_note)) - assert response.ok - response_content = response.json() - assert 'id' in response_content - assert 'created' in response_content - assert 'updated' in response_content - assert 'annotator_schema_version' in response_content - self.assertDictContainsSubset(dummy_note, response_content) - - def test_note_read(self): - notes = self._get_notes() - for note in notes: - response = requests.get(self._get_url("api/v1/annotations/" + note["id"])) - assert response.ok - self.assertDictEqual(note, response.json()) - - response = requests.get(self._get_url("api/v1/annotations/does_not_exist")) - assert response.status_code == 404 - - def test_note_update(self): - notes = self._get_notes() - for note in notes: - response = requests.get(self._get_url("api/v1/annotations/" + note["id"])) - assert response.ok - self.assertDictEqual(note, response.json()) - - response = requests.get(self._get_url("api/v1/annotations/does_not_exist")) - assert response.status_code == 404 - - def test_search(self): - # Without user - response = requests.get(self._get_url("api/v1/search")) - assert response.status_code == 400 - - # get response with default page and page size - response = requests.get(self._get_url("api/v1/search"), params={ - "user": "dummy-user-id", - "course_id": "dummy-course-id", - }) - - assert response.ok - self._verify_pagination_info( - response=response.json(), - total_notes=5, - num_pages=3, - notes_per_page=2, - start=0, - current_page=1, - next_page=2, - previous_page=None - ) - - # search notes with text that don't exist - response = requests.get(self._get_url("api/v1/search"), params={ - "user": "dummy-user-id", - "course_id": "dummy-course-id", - "text": "world war 2" - }) - - assert response.ok - self._verify_pagination_info( - response=response.json(), - total_notes=0, - num_pages=0, - notes_per_page=0, - start=0, - current_page=1, - next_page=None, - previous_page=None - ) - - @ddt.data( - '?usage_id=dummy-usage-id-0', - '?usage_id=dummy-usage-id-0&usage_id=dummy-usage-id-1&dummy-usage-id-2&dummy-usage-id-3&dummy-usage-id-4' - ) - def test_search_usage_ids(self, usage_ids): - """ - Test search with usage ids. - """ - url = self._get_url('api/v1/search') + usage_ids - response = requests.get(url, params={ - 'user': 'dummy-user-id', - 'course_id': 'dummy-course-id' - }) - assert response.ok - response = response.json() - parsed = six.moves.urllib.parse.urlparse(url) - query_params = six.moves.urllib.parse.parse_qs(parsed.query) - query_params['usage_id'].reverse() - assert len(response) == len(query_params['usage_id']) - for index, usage_id in enumerate(query_params['usage_id']): - assert response[index]['usage_id'] == usage_id - - def test_delete(self): - notes = self._get_notes() - response = requests.delete(self._get_url("api/v1/annotations/does_not_exist")) - assert response.status_code == 404 - - for note in notes: - response = requests.delete(self._get_url("api/v1/annotations/" + note["id"])) - assert response.status_code == 204 - remaining_notes = self.server.get_all_notes() - assert note['id'] not in [note['id'] for note in remaining_notes] - - assert len(remaining_notes) == 0 - - def test_update(self): - note = self._get_notes()[0] - response = requests.put(self._get_url("api/v1/annotations/" + note["id"]), data=json.dumps({ - "text": "new test text" - })) - assert response.status_code == 200 - - updated_note = self._get_notes()[0] - assert 'new test text' == updated_note['text'] - assert note['id'] == updated_note['id'] - self.assertCountEqual(note, updated_note) - - response = requests.get(self._get_url("api/v1/annotations/does_not_exist")) - assert response.status_code == 404 - - # pylint: disable=too-many-arguments - def _verify_pagination_info( - self, - response, - total_notes, - num_pages, - notes_per_page, - current_page, - previous_page, - next_page, - start - ): - """ - Verify the pagination information. - - Argument: - response: response from api - total_notes: total notes in the response - num_pages: total number of pages in response - notes_per_page: number of notes in the response - current_page: current page number - previous_page: previous page number - next_page: next page number - start: start of the current page - """ - def get_page_value(url): - """ - Return page value extracted from url. - """ - if url is None: - return None - - parsed = six.moves.urllib.parse.urlparse(url) - query_params = six.moves.urllib.parse.parse_qs(parsed.query) - - page = query_params["page"][0] - return page if page is None else int(page) - - assert response['total'] == total_notes - assert response['num_pages'] == num_pages - assert len(response['rows']) == notes_per_page - assert response['current_page'] == current_page - assert get_page_value(response['previous']) == previous_page - assert get_page_value(response['next']) == next_page - assert response['start'] == start - - def test_notes_collection(self): - """ - Test paginated response of notes api - """ - - # Without user - response = requests.get(self._get_url("api/v1/annotations")) - assert response.status_code == 400 - - # Without any pagination parameters - response = requests.get(self._get_url("api/v1/annotations"), params={"user": "dummy-user-id"}) - - assert response.ok - self._verify_pagination_info( - response=response.json(), - total_notes=5, - num_pages=3, - notes_per_page=2, - start=0, - current_page=1, - next_page=2, - previous_page=None - ) - - # With pagination parameters - response = requests.get(self._get_url("api/v1/annotations"), params={ - "user": "dummy-user-id", - "page": 2, - "page_size": 3 - }) - - assert response.ok - self._verify_pagination_info( - response=response.json(), - total_notes=5, - num_pages=2, - notes_per_page=2, - start=3, - current_page=2, - next_page=None, - previous_page=1 - ) - - def test_notes_collection_next_previous_with_one_page(self): - """ - Test next and previous urls of paginated response of notes api - when number of pages are 1 - """ - response = requests.get(self._get_url("api/v1/annotations"), params={ - "user": "dummy-user-id", - "page_size": 10 - }) - - assert response.ok - self._verify_pagination_info( - response=response.json(), - total_notes=5, - num_pages=1, - notes_per_page=5, - start=0, - current_page=1, - next_page=None, - previous_page=None - ) - - def test_notes_collection_when_no_notes(self): - """ - Test paginated response of notes api when there's no note present - """ - - # Delete all notes - self.test_cleanup() - - # Get default page - response = requests.get(self._get_url("api/v1/annotations"), params={"user": "dummy-user-id"}) - assert response.ok - self._verify_pagination_info( - response=response.json(), - total_notes=0, - num_pages=0, - notes_per_page=0, - start=0, - current_page=1, - next_page=None, - previous_page=None - ) - - def test_cleanup(self): - response = requests.put(self._get_url("cleanup")) - assert response.ok - assert len(self.server.get_all_notes()) == 0 - - def test_create_notes(self): - dummy_notes = self._get_dummy_notes(count=2) - response = requests.post(self._get_url("create_notes"), data=json.dumps(dummy_notes)) - assert response.ok - assert len(self._get_notes()) == 7 - - response = requests.post(self._get_url("create_notes")) - assert response.status_code == 400 - - def test_headers(self): - note = self._get_notes()[0] - response = requests.get(self._get_url("api/v1/annotations/" + note["id"])) - assert response.ok - assert response.headers.get('access-control-allow-origin') == '*' - - response = requests.options(self._get_url("api/v1/annotations/")) - assert response.ok - assert response.headers.get('access-control-allow-origin') == '*' - assert response.headers.get('access-control-allow-methods') == 'GET, POST, PUT, DELETE, OPTIONS' - assert 'X-CSRFToken' in response.headers.get('access-control-allow-headers') - - def _get_notes(self): - """ - Return a list of notes from the stub EdxNotes service. - """ - notes = self.server.get_all_notes() - assert len(notes) > 0, 'Notes are empty.' - return notes - - def _get_url(self, path): - """ - Construt a URL to the stub EdxNotes service. - """ - return "http://127.0.0.1:{port}/{path}/".format( - port=self.server.port, path=path - ) diff --git a/common/djangoapps/terrain/stubs/tests/test_http.py b/common/djangoapps/terrain/stubs/tests/test_http.py deleted file mode 100644 index f9f5125c1f2b..000000000000 --- a/common/djangoapps/terrain/stubs/tests/test_http.py +++ /dev/null @@ -1,124 +0,0 @@ -""" -Unit tests for stub HTTP server base class. -""" - - -import json -import unittest - -import requests - -from common.djangoapps.terrain.stubs.http import StubHttpRequestHandler, StubHttpService, require_params - - -class StubHttpServiceTest(unittest.TestCase): # lint-amnesty, pylint: disable=missing-class-docstring - - def setUp(self): - super().setUp() - self.server = StubHttpService() - self.addCleanup(self.server.shutdown) - self.url = f"http://127.0.0.1:{self.server.port}/set_config" - - def test_configure(self): - """ - All HTTP stub servers have an end-point that allows - clients to configure how the server responds. - """ - params = { - 'test_str': 'This is only a test', - 'test_empty': '', - 'test_int': 12345, - 'test_float': 123.45, - 'test_dict': { - 'test_key': 'test_val', - }, - 'test_empty_dict': {}, - 'test_unicode': '\u2603 the snowman', - 'test_none': None, - 'test_boolean': False - } - - for key, val in params.items(): - - # JSON-encode each parameter - post_params = {key: json.dumps(val)} - response = requests.put(self.url, data=post_params) - assert response.status_code == 200 - - # Check that the expected values were set in the configuration - for key, val in params.items(): - assert self.server.config.get(key) == val - - def test_bad_json(self): - response = requests.put(self.url, data="{,}") - assert response.status_code == 400 - - def test_no_post_data(self): - response = requests.put(self.url, data={}) - assert response.status_code == 200 - - def test_unicode_non_json(self): - # Send unicode without json-encoding it - response = requests.put(self.url, data={'test_unicode': '\u2603 the snowman'}) - assert response.status_code == 400 - - def test_unknown_path(self): - response = requests.put( - f"http://127.0.0.1:{self.server.port}/invalid_url", - data="{}" - ) - assert response.status_code == 404 - - -class RequireRequestHandler(StubHttpRequestHandler): # lint-amnesty, pylint: disable=missing-class-docstring - @require_params('GET', 'test_param') - def do_GET(self): - self.send_response(200) - - @require_params('POST', 'test_param') - def do_POST(self): - self.send_response(200) - - -class RequireHttpService(StubHttpService): - HANDLER_CLASS = RequireRequestHandler - - -class RequireParamTest(unittest.TestCase): - """ - Test the decorator for requiring parameters. - """ - - def setUp(self): - super().setUp() - self.server = RequireHttpService() - self.addCleanup(self.server.shutdown) - self.url = f"http://127.0.0.1:{self.server.port}" - - def test_require_get_param(self): - - # Expect success when we provide the required param - response = requests.get(self.url, params={"test_param": 2}) - assert response.status_code == 200 - - # Expect failure when we do not proivde the param - response = requests.get(self.url) - assert response.status_code == 400 - - # Expect failure when we provide an empty param - response = requests.get(self.url + "?test_param=") - assert response.status_code == 400 - - def test_require_post_param(self): - - # Expect success when we provide the required param - response = requests.post(self.url, data={"test_param": 2}) - assert response.status_code == 200 - - # Expect failure when we do not proivde the param - response = requests.post(self.url) - assert response.status_code == 400 - - # Expect failure when we provide an empty param - response = requests.post(self.url, data={"test_param": None}) - assert response.status_code == 400 diff --git a/common/djangoapps/terrain/stubs/tests/test_lti_stub.py b/common/djangoapps/terrain/stubs/tests/test_lti_stub.py deleted file mode 100644 index 1d04f47cd9df..000000000000 --- a/common/djangoapps/terrain/stubs/tests/test_lti_stub.py +++ /dev/null @@ -1,98 +0,0 @@ -""" -Unit tests for stub LTI implementation. -""" - - -import unittest -from unittest.mock import Mock, patch - -import requests -from urllib.request import urlopen # pylint: disable=wrong-import-order - -from common.djangoapps.terrain.stubs.lti import StubLtiService - - -class StubLtiServiceTest(unittest.TestCase): - """ - A stub of the LTI provider that listens on a local - port and responds with pre-defined grade messages. - - Used for lettuce BDD tests in lms/courseware/features/lti.feature - """ - def setUp(self): - super().setUp() - self.server = StubLtiService() - self.uri = f'http://127.0.0.1:{self.server.port}/' - self.launch_uri = self.uri + 'correct_lti_endpoint' - self.addCleanup(self.server.shutdown) - self.payload = { - 'user_id': 'default_user_id', - 'roles': 'Student', - 'oauth_nonce': '', - 'oauth_timestamp': '', - 'oauth_consumer_key': 'test_client_key', - 'lti_version': 'LTI-1p0', - 'oauth_signature_method': 'HMAC-SHA1', - 'oauth_version': '1.0', - 'oauth_signature': '', - 'lti_message_type': 'basic-lti-launch-request', - 'oauth_callback': 'about:blank', - 'launch_presentation_return_url': '', - 'lis_outcome_service_url': 'http://localhost:8001/test_callback', - 'lis_result_sourcedid': '', - 'resource_link_id': '', - } - - def test_invalid_request_url(self): - """ - Tests that LTI server processes request with right program path but with wrong header. - """ - self.launch_uri = self.uri + 'wrong_lti_endpoint' - response = requests.post(self.launch_uri, data=self.payload) - assert b'Invalid request URL' in response.content - - def test_wrong_signature(self): - """ - Tests that LTI server processes request with right program - path and responses with incorrect signature. - """ - response = requests.post(self.launch_uri, data=self.payload) - assert b'Wrong LTI signature' in response.content - - @patch('common.djangoapps.terrain.stubs.lti.signature.verify_hmac_sha1', return_value=True) - def test_success_response_launch_lti(self, check_oauth): # lint-amnesty, pylint: disable=unused-argument - """ - Success lti launch. - """ - response = requests.post(self.launch_uri, data=self.payload) - assert b'This is LTI tool. Success.' in response.content - - @patch('common.djangoapps.terrain.stubs.lti.signature.verify_hmac_sha1', return_value=True) - def test_send_graded_result(self, verify_hmac): # pylint: disable=unused-argument - response = requests.post(self.launch_uri, data=self.payload) - assert b'This is LTI tool. Success.' in response.content - grade_uri = self.uri + 'grade' - with patch('common.djangoapps.terrain.stubs.lti.requests.post') as mocked_post: - mocked_post.return_value = Mock(content='Test response', status_code=200) - response = urlopen(grade_uri, data=b'') # lint-amnesty, pylint: disable=consider-using-with - assert b'Test response' in response.read() - - @patch('common.djangoapps.terrain.stubs.lti.signature.verify_hmac_sha1', return_value=True) - def test_lti20_outcomes_put(self, verify_hmac): # pylint: disable=unused-argument - response = requests.post(self.launch_uri, data=self.payload) - assert b'This is LTI tool. Success.' in response.content - grade_uri = self.uri + 'lti2_outcome' - with patch('common.djangoapps.terrain.stubs.lti.requests.put') as mocked_put: - mocked_put.return_value = Mock(status_code=200) - response = urlopen(grade_uri, data=b'') # lint-amnesty, pylint: disable=consider-using-with - assert b'LTI consumer (edX) responded with HTTP 200' in response.read() - - @patch('common.djangoapps.terrain.stubs.lti.signature.verify_hmac_sha1', return_value=True) - def test_lti20_outcomes_put_like_delete(self, verify_hmac): # pylint: disable=unused-argument - response = requests.post(self.launch_uri, data=self.payload) - assert b'This is LTI tool. Success.' in response.content - grade_uri = self.uri + 'lti2_delete' - with patch('common.djangoapps.terrain.stubs.lti.requests.put') as mocked_put: - mocked_put.return_value = Mock(status_code=200) - response = urlopen(grade_uri, data=b'') # lint-amnesty, pylint: disable=consider-using-with - assert b'LTI consumer (edX) responded with HTTP 200' in response.read() diff --git a/common/djangoapps/terrain/stubs/tests/test_video.py b/common/djangoapps/terrain/stubs/tests/test_video.py deleted file mode 100644 index 66332bdf0e87..000000000000 --- a/common/djangoapps/terrain/stubs/tests/test_video.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -Unit tests for Video stub server implementation. -""" - - -import unittest - -import requests -from django.conf import settings - -from common.djangoapps.terrain.stubs.video_source import VideoSourceHttpService - -HLS_MANIFEST_TEXT = """ -#EXTM3U -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=264787,RESOLUTION=1280x720 -history_264kbit/history_264kbit.m3u8 -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=328415,RESOLUTION=1920x1080 -history_328kbit/history_328kbit.m3u8 -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=70750,RESOLUTION=640x360 -history_70kbit/history_70kbit.m3u8 -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=148269,RESOLUTION=960x540 -history_148kbit/history_148kbit.m3u8 -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=41276,RESOLUTION=640x360 -history_41kbit/history_41kbit.m3u8 -""" - - -class StubVideoServiceTest(unittest.TestCase): - """ - Test cases for the video stub service. - """ - def setUp(self): - """ - Start the stub server. - """ - super().setUp() - self.server = VideoSourceHttpService() - self.server.config['root_dir'] = f'{settings.TEST_ROOT}/data/video' - self.addCleanup(self.server.shutdown) - - def test_get_hls_manifest(self): - """ - Verify that correct hls manifest is received. - """ - response = requests.get(f"http://127.0.0.1:{self.server.port}/hls/history.m3u8") - assert response.ok - assert response.text == HLS_MANIFEST_TEXT.lstrip() - assert response.headers['Access-Control-Allow-Origin'] == '*' diff --git a/common/djangoapps/terrain/stubs/tests/test_xqueue_stub.py b/common/djangoapps/terrain/stubs/tests/test_xqueue_stub.py deleted file mode 100644 index 5a4576817ec1..000000000000 --- a/common/djangoapps/terrain/stubs/tests/test_xqueue_stub.py +++ /dev/null @@ -1,173 +0,0 @@ -""" -Unit tests for stub XQueue implementation. -""" - - -import ast -import json -import unittest -from unittest import mock - -import requests - -from ..xqueue import StubXQueueService - - -class FakeTimer: - """ - Fake timer implementation that executes immediately. - """ - def __init__(self, delay, func): # lint-amnesty, pylint: disable=unused-argument - self.func = func - - def start(self): - self.func() - - -class StubXQueueServiceTest(unittest.TestCase): # lint-amnesty, pylint: disable=missing-class-docstring - - def setUp(self): - super().setUp() - self.server = StubXQueueService() - self.url = f"http://127.0.0.1:{self.server.port}/xqueue/submit" - self.addCleanup(self.server.shutdown) - - # Patch the timer async calls - patcher = mock.patch('common.djangoapps.terrain.stubs.xqueue.post') - self.post = patcher.start() - self.addCleanup(patcher.stop) - - # Patch POST requests - patcher = mock.patch('common.djangoapps.terrain.stubs.xqueue.Timer') - timer = patcher.start() - timer.side_effect = FakeTimer - self.addCleanup(patcher.stop) - - def test_grade_request(self): - - # Post a submission to the stub XQueue - callback_url = 'http://127.0.0.1:8000/test_callback' - expected_header = self._post_submission( - callback_url, 'test_queuekey', 'test_queue', - json.dumps({ - 'student_info': 'test', - 'grader_payload': 'test', - 'student_response': 'test' - }) - ) - - # Check the response we receive - # (Should be the default grading response) - expected_body = json.dumps({'correct': True, 'score': 1, 'msg': '
'}) - self._check_grade_response(callback_url, expected_header, expected_body) - - def test_configure_default_response(self): - - # Configure the default response for submissions to any queue - response_content = {'test_response': 'test_content'} - self.server.config['default'] = response_content - - # Post a submission to the stub XQueue - callback_url = 'http://127.0.0.1:8000/test_callback' - expected_header = self._post_submission( - callback_url, 'test_queuekey', 'test_queue', - json.dumps({ - 'student_info': 'test', - 'grader_payload': 'test', - 'student_response': 'test' - }) - ) - - # Check the response we receive - # (Should be the default grading response) - self._check_grade_response(callback_url, expected_header, json.dumps(response_content)) - - def test_configure_specific_response(self): - - # Configure the XQueue stub response to any submission to the test queue - response_content = {'test_response': 'test_content'} - self.server.config['This is only a test.'] = response_content - - # Post a submission to the XQueue stub - callback_url = 'http://127.0.0.1:8000/test_callback' - expected_header = self._post_submission( - callback_url, 'test_queuekey', 'test_queue', - json.dumps({'submission': 'This is only a test.'}) - ) - - # Check that we receive the response we configured - self._check_grade_response(callback_url, expected_header, json.dumps(response_content)) - - def test_multiple_response_matches(self): - - # Configure the XQueue stub with two responses that - # match the same submission - self.server.config['test_1'] = {'response': True} - self.server.config['test_2'] = {'response': False} - - with mock.patch('common.djangoapps.terrain.stubs.http.LOGGER') as logger: - - # Post a submission to the XQueue stub - callback_url = 'http://127.0.0.1:8000/test_callback' - self._post_submission( - callback_url, 'test_queuekey', 'test_queue', - json.dumps({'submission': 'test_1 and test_2'}) - ) - - # Expect that we do NOT receive a response - # and that an error message is logged - assert not self.post.called - assert logger.error.called - - def _post_submission(self, callback_url, lms_key, queue_name, xqueue_body): # lint-amnesty, pylint: disable=unused-argument - """ - Post a submission to the stub XQueue implementation. - `callback_url` is the URL at which we expect to receive a grade response - `lms_key` is the authentication key sent in the header - `queue_name` is the name of the queue in which to send put the submission - `xqueue_body` is the content of the submission - - Returns the header (a string) we send with the submission, which can - be used to validate the response we receive from the stub. - """ - - # Post a submission to the XQueue stub - grade_request = { - 'xqueue_header': json.dumps({ - 'lms_callback_url': callback_url, - 'lms_key': 'test_queuekey', - 'queue_name': 'test_queue' - }), - 'xqueue_body': xqueue_body - } - - resp = requests.post(self.url, data=grade_request) - - # Expect that the response is success - assert resp.status_code == 200 - - # Return back the header, so we can authenticate the response we receive - return grade_request['xqueue_header'] - - def _check_grade_response(self, callback_url, expected_header, expected_body): - """ - Verify that the stub sent a POST request back to us - with the expected data. - - `callback_url` is the URL we expect the stub to POST to - `expected_header` is the header (a string) we expect to receive with the grade. - `expected_body` is the content (a string) we expect to receive with the grade. - - Raises an `AssertionError` if the check fails. - """ - # Check the response posted back to us - # This is the default response - expected_callback_dict = { - 'xqueue_header': expected_header, - 'xqueue_body': expected_body, - } - # Check that the POST request was made with the correct params - assert self.post.call_args[1]['data']['xqueue_body'] == expected_callback_dict['xqueue_body'] - assert ast.literal_eval(self.post.call_args[1]['data']['xqueue_header']) ==\ - ast.literal_eval(expected_callback_dict['xqueue_header']) - assert self.post.call_args[0][0] == callback_url diff --git a/common/djangoapps/terrain/stubs/tests/test_youtube_stub.py b/common/djangoapps/terrain/stubs/tests/test_youtube_stub.py deleted file mode 100644 index 74c613dfe2ef..000000000000 --- a/common/djangoapps/terrain/stubs/tests/test_youtube_stub.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -Unit test for stub YouTube implementation. -""" - - -import unittest - -import requests - -from ..youtube import StubYouTubeService - - -class StubYouTubeServiceTest(unittest.TestCase): # lint-amnesty, pylint: disable=missing-class-docstring - - def setUp(self): - super().setUp() - self.server = StubYouTubeService() - self.url = f"http://127.0.0.1:{self.server.port}/" - self.server.config['time_to_response'] = 0.0 - self.addCleanup(self.server.shutdown) - - def test_unused_url(self): - response = requests.get(self.url + 'unused_url') - assert b'Unused url' == response.content - - @unittest.skip('Failing intermittently due to inconsistent responses from YT. See TE-871') - def test_video_url(self): - response = requests.get( - self.url + 'test_youtube/OEoXaMPEzfM?v=2&alt=jsonc&callback=callback_func' - ) - - # YouTube metadata for video `OEoXaMPEzfM` states that duration is 116. - assert b'callback_func({"data": {"duration": 116, "message": "I\'m youtube.", "id": "OEoXaMPEzfM"}})' ==\ - response.content - - def test_transcript_url_equal(self): - response = requests.get( - self.url + 'test_transcripts_youtube/t__eq_exist' - ) - - assert ''.join(['', - '', - 'Equal transcripts']).encode('utf-8') == response.content - - def test_transcript_url_not_equal(self): - response = requests.get( - self.url + 'test_transcripts_youtube/t_neq_exist', - ) - - assert ''.join(['', - '', - 'Transcripts sample, different that on server', - '']).encode('utf-8') == response.content - - def test_transcript_not_found(self): - response = requests.get(self.url + 'test_transcripts_youtube/some_id') - assert 404 == response.status_code - - def test_reset_configuration(self): - - reset_config_url = self.url + 'del_config' - - # add some configuration data - self.server.config['test_reset'] = 'This is a reset config test' - - # reset server configuration - response = requests.delete(reset_config_url) - assert response.status_code == 200 - - # ensure that server config dict is empty after successful reset - assert not self.server.config diff --git a/common/djangoapps/terrain/stubs/video_source.py b/common/djangoapps/terrain/stubs/video_source.py deleted file mode 100644 index ffff090a624f..000000000000 --- a/common/djangoapps/terrain/stubs/video_source.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Serve HTML5 video sources for acceptance tests -""" - - -import os -from contextlib import contextmanager -from logging import getLogger - -from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler - -from .http import StubHttpService - -LOGGER = getLogger(__name__) - - -class VideoSourceRequestHandler(SimpleHTTPRequestHandler): - """ - Request handler for serving video sources locally. - """ - def translate_path(self, path): - """ - Remove any extra parameters from the path. - For example /gizmo.mp4?1397160769634 - becomes /gizmo.mp4 - """ - root_dir = self.server.config.get('root_dir') - path = f'{root_dir}{path}' - return path.split('?')[0] - - def end_headers(self): - """ - This is required by hls.js to play hls videos. - """ - self.send_header('Access-Control-Allow-Origin', '*') - SimpleHTTPRequestHandler.end_headers(self) - - -class VideoSourceHttpService(StubHttpService): - """ - Simple HTTP server for serving HTML5 Video sources locally for tests - """ - HANDLER_CLASS = VideoSourceRequestHandler - - def __init__(self, port_num=0): - - @contextmanager - def _remember_cwd(): - """ - Files are automatically served from the current directory - so we need to change it, start the server, then set it back. - """ - curdir = os.getcwd() - try: - yield - finally: - os.chdir(curdir) - - with _remember_cwd(): - StubHttpService.__init__(self, port_num=port_num) diff --git a/common/djangoapps/terrain/stubs/xqueue.py b/common/djangoapps/terrain/stubs/xqueue.py deleted file mode 100644 index bb6b7ac69efc..000000000000 --- a/common/djangoapps/terrain/stubs/xqueue.py +++ /dev/null @@ -1,226 +0,0 @@ -""" -Stub implementation of XQueue for acceptance tests. - -Configuration values: - "default" (dict): Default response to be sent to LMS as a grade for a submission - "" (dict): Grade response to return for submissions containing the text - "register_submission_url" (str): URL to send grader payloads when we receive a submission - -If no grade response is configured, a default response will be returned. -""" - - -import copy -import json -from threading import Timer - -from requests import post - -from openedx.core.djangolib.markup import HTML - -from .http import StubHttpRequestHandler, StubHttpService, require_params - - -class StubXQueueHandler(StubHttpRequestHandler): - """ - A handler for XQueue POST requests. - """ - - DEFAULT_RESPONSE_DELAY = 2 - DEFAULT_GRADE_RESPONSE = {'correct': True, 'score': 1, 'msg': ''} - - @require_params('POST', 'xqueue_body', 'xqueue_header') - def do_POST(self): - """ - Handle a POST request from the client - - Sends back an immediate success/failure response. - It then POSTS back to the client with grading results. - """ - msg = f"XQueue received POST request {self.post_dict} to path {self.path}" - self.log_message(msg) - - # Respond only to grading requests - if self._is_grade_request(): - - # If configured, send the grader payload to other services. - # TODO TNL-3906 - # self._register_submission(self.post_dict['xqueue_body']) - - try: - xqueue_header = json.loads(self.post_dict['xqueue_header']) - callback_url = xqueue_header['lms_callback_url'] - - except KeyError: - # If the message doesn't have a header or body, - # then it's malformed. Respond with failure - error_msg = "XQueue received invalid grade request" - self._send_immediate_response(False, message=error_msg) - - except ValueError: - # If we could not decode the body or header, - # respond with failure - error_msg = "XQueue could not decode grade request" - self._send_immediate_response(False, message=error_msg) - - else: - # Send an immediate response of success - # The grade request is formed correctly - self._send_immediate_response(True) - - # Wait a bit before POSTing back to the callback url with the - # grade result configured by the server - # Otherwise, the problem will not realize it's - # queued and it will keep waiting for a response indefinitely - delayed_grade_func = lambda: self._send_grade_response( - callback_url, xqueue_header, self.post_dict['xqueue_body'] - ) - - delay = self.server.config.get('response_delay', self.DEFAULT_RESPONSE_DELAY) - Timer(delay, delayed_grade_func).start() - - # If we get a request that's not to the grading submission - # URL, return an error - else: - self._send_immediate_response(False, message="Invalid request URL") - - def _send_immediate_response(self, success, message=""): - """ - Send an immediate success/failure message - back to the client - """ - - # Send the response indicating success/failure - response_str = json.dumps( - {'return_code': 0 if success else 1, 'content': message} - ) - - if self._is_grade_request(): - self.send_response( - 200, content=response_str, headers={'Content-type': 'text/plain'} - ) - self.log_message(f"XQueue: sent response {response_str}") - - else: - self.send_response(500) - - def _send_grade_response(self, postback_url, xqueue_header, xqueue_body_json): - """ - POST the grade response back to the client - using the response provided by the server configuration. - - Uses the server configuration to determine what response to send: - 1) Specific response for submissions containing matching text in `xqueue_body` - 2) Default submission configured by client - 3) Default submission - - `postback_url` is the URL the client told us to post back to - `xqueue_header` (dict) is the full header the client sent us, which we will send back - to the client so it can authenticate us. - `xqueue_body_json` (json-encoded string) is the body of the submission the client sent us. - """ - # First check if we have a configured response that matches the submission body - grade_response = None - - # This matches the pattern against the JSON-encoded xqueue_body - # This is very simplistic, but sufficient to associate a student response - # with a grading response. - # There is a danger here that a submission will match multiple response patterns. - # Rather than fail silently (which could cause unpredictable behavior in tests) - # we abort and log a debugging message. - for pattern, response in self.server.queue_responses: - - if pattern in xqueue_body_json: - if grade_response is None: - grade_response = response - - # Multiple matches, so abort and log an error - else: - self.log_error( - f"Multiple response patterns matched '{xqueue_body_json}'", - ) - return - - # Fall back to the default grade response configured for this queue, - # then to the default response. - if grade_response is None: - grade_response = self.server.config.get( - 'default', copy.deepcopy(self.DEFAULT_GRADE_RESPONSE) - ) - - # Wrap the message in
tags to ensure that it is valid XML - if isinstance(grade_response, dict) and 'msg' in grade_response: - grade_response['msg'] = HTML("
{0}
").format(grade_response['msg']) - - data = { - 'xqueue_header': json.dumps(xqueue_header), - 'xqueue_body': json.dumps(grade_response) - } - - post(postback_url, data=data) - self.log_message(f"XQueue: sent grading response {data} to {postback_url}") - - def _register_submission(self, xqueue_body_json): - """ - If configured, send the submission's grader payload to another service. - """ - url = self.server.config.get('register_submission_url') - - # If not configured, do not need to send anything - if url is not None: - - try: - xqueue_body = json.loads(xqueue_body_json) - except ValueError: - self.log_error( - f"Could not decode XQueue body as JSON: '{xqueue_body_json}'") - - else: - - # Retrieve the grader payload, which should be a JSON-encoded dict. - # We pass the payload directly to the service we are notifying, without - # inspecting the contents. - grader_payload = xqueue_body.get('grader_payload') - - if grader_payload is not None: - response = post(url, data={'grader_payload': grader_payload}) - if not response.ok: - self.log_error( - "Could register submission at URL '{}'. Status was {}".format( - url, response.status_code)) - - else: - self.log_message( - f"XQueue body is missing 'grader_payload' key: '{xqueue_body}'" - ) - - def _is_grade_request(self): - """ - Return a boolean indicating whether the requested URL indicates a submission. - """ - return 'xqueue/submit' in self.path - - -class StubXQueueService(StubHttpService): - """ - A stub XQueue grading server that responds to POST requests to localhost. - """ - - HANDLER_CLASS = StubXQueueHandler - NON_QUEUE_CONFIG_KEYS = ['default', 'register_submission_url'] - - @property - def queue_responses(self): - """ - Returns a list of (pattern, response) tuples, where `pattern` is a pattern - to match in the XQueue body, and `response` is a dictionary to return - as the response from the grader. - - Every configuration key is a queue name, - except for 'default' and 'register_submission_url' which have special meaning - """ - return list({ - key: value - for key, value in self.config.items() - if key not in self.NON_QUEUE_CONFIG_KEYS - }.items()) diff --git a/common/djangoapps/terrain/stubs/youtube.py b/common/djangoapps/terrain/stubs/youtube.py deleted file mode 100644 index 67cac950f9fb..000000000000 --- a/common/djangoapps/terrain/stubs/youtube.py +++ /dev/null @@ -1,172 +0,0 @@ -""" -Stub implementation of YouTube for acceptance tests. - - -To start this stub server on its own from Vagrant: - -1.) Locally, modify your Vagrantfile so that it contains: - - config.vm.network :forwarded_port, guest: 8031, host: 8031 - -2.) From within Vagrant dev environment do: - - cd common/djangoapps/terrain - python -m stubs.start youtube 8031 - -3.) Locally, try accessing http://localhost:8031/ and see that - you get "Unused url" message inside the browser. -""" - - -import json -import time -from collections import OrderedDict - -import requests -from six.moves.urllib.parse import urlparse - -from .http import StubHttpRequestHandler, StubHttpService - - -class StubYouTubeHandler(StubHttpRequestHandler): - """ - A handler for Youtube GET requests. - """ - - # Default number of seconds to delay the response to simulate network latency. - DEFAULT_DELAY_SEC = 0.5 - - def do_DELETE(self): # pylint: disable=invalid-name - """ - Allow callers to delete all the server configurations using the /del_config URL. - """ - if self.path in ("/del_config", "/del_config/"): - self.server.config = {} - self.log_message("Reset Server Configuration.") - self.send_response(200) - else: - self.send_response(404) - - def do_GET(self): - """ - Handle a GET request from the client and sends response back. - """ - self.log_message( - f"Youtube provider received GET request to path {self.path}" - ) - - if 'get_config' in self.path: - self.send_json_response(self.server.config) - - elif 'test_transcripts_youtube' in self.path: - - if 't__eq_exist' in self.path: - status_message = "".join([ - '', - '', - 'Equal transcripts' - ]).encode('utf-8') - - self.send_response( - 200, content=status_message, headers={'Content-type': 'application/xml'} - ) - - elif 't_neq_exist' in self.path: - status_message = "".join([ - '', - '', - 'Transcripts sample, different that on server', - '' - ]).encode('utf-8') - - self.send_response( - 200, content=status_message, headers={'Content-type': 'application/xml'} - ) - - else: - self.send_response(404) - - elif 'test_youtube' in self.path: - params = urlparse(self.path) - youtube_id = params.path.split('/').pop() - - if self.server.config.get('youtube_api_private_video'): - self._send_private_video_response(youtube_id, "I'm youtube private video.") # lint-amnesty, pylint: disable=too-many-function-args - else: - self._send_video_response(youtube_id, "I'm youtube.") - - elif 'get_youtube_api' in self.path: - # Delay the response to simulate network latency - time.sleep(self.server.config.get('time_to_response', self.DEFAULT_DELAY_SEC)) - if self.server.config.get('youtube_api_blocked'): - self.send_response(404, content=b'', headers={'Content-type': 'text/plain'}) - else: - # Get the response to send from YouTube. - # We need to do this every time because Google sometimes sends different responses - # as part of their own experiments, which has caused our tests to become "flaky" - self.log_message("Getting iframe api from youtube.com") - iframe_api_response = requests.get('https://www.youtube.com/iframe_api').content.strip(b"\n") - self.send_response(200, content=iframe_api_response, headers={'Content-type': 'text/html'}) - - else: - self.send_response( - 404, content=b"Unused url", headers={'Content-type': 'text/plain'} - ) - - def _send_video_response(self, youtube_id, message): - """ - Send message back to the client for video player requests. - Requires sending back callback id. - """ - # Delay the response to simulate network latency - time.sleep(self.server.config.get('time_to_response', self.DEFAULT_DELAY_SEC)) - - # Construct the response content - callback = self.get_params['callback'] - - data = OrderedDict({ - 'items': list( - OrderedDict({ - 'contentDetails': OrderedDict({ - 'id': youtube_id, - 'duration': 'PT2M20S', - }) - }) - ) - }) - response = f"{callback}({json.dumps(data)})".encode('utf-8') - - self.send_response(200, content=response, headers={'Content-type': 'text/html'}) - self.log_message(f"Youtube: sent response {message}") - - def _send_private_video_response(self, message): - """ - Send private video error message back to the client for video player requests. - """ - # Construct the response content - callback = self.get_params['callback'] - data = OrderedDict({ - "error": OrderedDict({ - "code": 403, - "errors": [ - { - "code": "ServiceForbiddenException", - "domain": "GData", - "internalReason": "Private video" - } - ], - "message": message, - }) - }) - response = f"{callback}({json.dumps(data)})".encode('utf-8') - - self.send_response(200, content=response, headers={'Content-type': 'text/html'}) - self.log_message(f"Youtube: sent response {message}") - - -class StubYouTubeService(StubHttpService): - """ - A stub Youtube provider server that responds to GET requests to localhost. - """ - - HANDLER_CLASS = StubYouTubeHandler diff --git a/conf/locale/config.yaml b/conf/locale/config.yaml index 6faf3188be1f..8a486ec626d3 100644 --- a/conf/locale/config.yaml +++ b/conf/locale/config.yaml @@ -26,7 +26,6 @@ ignore_dirs: # Directories that only contain tests. - common/test - test_root - - '*/terrain' - '*/spec' - '*/tests' - '*/djangoapps/*/features' diff --git a/scripts/vulture/find-dead-code.sh b/scripts/vulture/find-dead-code.sh index e24882595ebb..c78793ec3a6d 100755 --- a/scripts/vulture/find-dead-code.sh +++ b/scripts/vulture/find-dead-code.sh @@ -35,9 +35,9 @@ mkdir -p "$OUTPUT_DIR" OUTPUT_FILE="${OUTPUT_DIR}/vulture-report.txt" echo '' > "$OUTPUT_FILE" # exclude test code from analysis, as it isn't explicitly called by other -# code. Additionally, application code that is only called by tests +# code. Additionally, application code that is only called by tests # should be considered dead -EXCLUSIONS='/test,/acceptance,cms/envs,lms/envs,/terrain,migrations/,signals.py' +EXCLUSIONS='/test,/acceptance,cms/envs,lms/envs,migrations/,signals.py' MIN_CONFIDENCE=90 # paths to the code on which to run the analysis CODE_PATHS=('cms' 'common' 'lms' 'openedx') From 838977a8f3981be491a7fa2441724e4e6e92d895 Mon Sep 17 00:00:00 2001 From: Muhammad Arslan Date: Wed, 1 Jan 2025 20:18:10 +0500 Subject: [PATCH 04/18] feat!: Remove DEPR waffle switch: ENABLE_GLOBAL_STAFF_OPTIMIZATION --- .../rest_api/v1/serializers/home.py | 1 - .../contentstore/rest_api/v1/views/home.py | 1 - .../rest_api/v1/views/tests/test_home.py | 23 ----------------- .../rest_api/v2/views/tests/test_home.py | 25 ------------------- cms/djangoapps/contentstore/utils.py | 16 ++---------- cms/djangoapps/contentstore/views/course.py | 20 +++------------ 6 files changed, 6 insertions(+), 80 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py b/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py index fa2a651f8a28..36166e42a5a6 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py @@ -62,7 +62,6 @@ class StudioHomeSerializer(serializers.Serializer): libraries_v2_enabled = serializers.BooleanField() taxonomies_enabled = serializers.BooleanField() taxonomy_list_mfe_url = serializers.CharField() - optimization_enabled = serializers.BooleanField() request_course_creator_url = serializers.CharField() rerun_creator_status = serializers.BooleanField() show_new_library_button = serializers.BooleanField() diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/home.py b/cms/djangoapps/contentstore/rest_api/v1/views/home.py index 3de536d78092..2d5360d38d6c 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/home.py @@ -62,7 +62,6 @@ def get(self, request: Request): "libraries_v1_enabled": true, "libraries_v2_enabled": true, "library_authoring_mfe_url": "//localhost:3001/course/course-v1:edX+P315+2T2023", - "optimization_enabled": true, "request_course_creator_url": "/request_course_creator", "rerun_creator_status": true, "show_new_library_button": true, diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py index a4a6909c5dcb..e78558bba330 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py @@ -15,7 +15,6 @@ from cms.djangoapps.contentstore.tests.utils import CourseTestCase from cms.djangoapps.contentstore.tests.test_libraries import LibraryTestCase -from cms.djangoapps.contentstore.views.course import ENABLE_GLOBAL_STAFF_OPTIMIZATION from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory from xmodule.modulestore.tests.factories import CourseFactory @@ -52,7 +51,6 @@ def setUp(self): "libraries_v2_enabled": False, "taxonomies_enabled": True, "taxonomy_list_mfe_url": 'http://course-authoring-mfe/taxonomies', - "optimization_enabled": False, "request_course_creator_url": "/request_course_creator", "rerun_creator_status": True, "show_new_library_button": True, @@ -242,27 +240,6 @@ def test_home_page_response_no_courses_non_staff(self, filter_key, filter_value) self.assertEqual(len(response.data["courses"]), 0) self.assertEqual(response.status_code, status.HTTP_200_OK) - @override_waffle_switch(ENABLE_GLOBAL_STAFF_OPTIMIZATION, True) - def test_org_query_if_passed(self): - """Test home page when org filter passed as a query param""" - foo_course = self.store.make_course_key('foo-org', 'bar-number', 'baz-run') - test_course = CourseFactory.create( - org=foo_course.org, - number=foo_course.course, - run=foo_course.run - ) - CourseOverviewFactory.create(id=test_course.id, org='foo-org') - response = self.client.get(self.url, {"org": "foo-org"}) - self.assertEqual(len(response.data['courses']), 1) - self.assertEqual(response.status_code, status.HTTP_200_OK) - - @override_waffle_switch(ENABLE_GLOBAL_STAFF_OPTIMIZATION, True) - def test_org_query_if_empty(self): - """Test home page with an empty org query param""" - response = self.client.get(self.url) - self.assertEqual(len(response.data['courses']), 0) - self.assertEqual(response.status_code, status.HTTP_200_OK) - @ddt.ddt class HomePageLibrariesViewTest(LibraryTestCase): diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_home.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_home.py index e773e7f213c6..bf3332e1e3cd 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_home.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_home.py @@ -15,7 +15,6 @@ from cms.djangoapps.contentstore.tests.utils import CourseTestCase from cms.djangoapps.contentstore.utils import reverse_course_url -from cms.djangoapps.contentstore.views.course import ENABLE_GLOBAL_STAFF_OPTIMIZATION from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory FEATURES_WITH_HOME_PAGE_COURSE_V2_API = settings.FEATURES.copy() @@ -104,30 +103,6 @@ def test_home_page_response(self): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertDictEqual(expected_response, response.data) - @override_waffle_switch(ENABLE_GLOBAL_STAFF_OPTIMIZATION, True) - def test_org_query_if_passed(self): - """Get list of courses when org filter passed as a query param. - - Expected result: - - A list of courses available to the logged in user for the specified org. - """ - response = self.client.get(self.api_v2_url, {"org": "demo-org"}) - - self.assertEqual(len(response.data['results']['courses']), 1) - self.assertEqual(response.status_code, status.HTTP_200_OK) - - @override_waffle_switch(ENABLE_GLOBAL_STAFF_OPTIMIZATION, True) - def test_org_query_if_empty(self): - """Get home page with an empty org query param. - - Expected result: - - An empty list of courses available to the logged in user. - """ - response = self.client.get(self.api_v2_url) - - self.assertEqual(len(response.data['results']['courses']), 0) - self.assertEqual(response.status_code, status.HTTP_200_OK) - def test_active_only_query_if_passed(self): """Get list of active courses only. diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index 7023bcaefaf7..a220b8d91399 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -1595,7 +1595,6 @@ def get_course_context(request): from cms.djangoapps.contentstore.views.course import ( get_courses_accessible_to_user, _process_courses_list, - ENABLE_GLOBAL_STAFF_OPTIMIZATION, ) def format_in_process_course_view(uca): @@ -1619,10 +1618,7 @@ def format_in_process_course_view(uca): ) if uca.state == CourseRerunUIStateManager.State.FAILED else '' } - optimization_enabled = GlobalStaff().has_user(request.user) and ENABLE_GLOBAL_STAFF_OPTIMIZATION.is_enabled() - - org = request.GET.get('org', '') if optimization_enabled else None - courses_iter, in_process_course_actions = get_courses_accessible_to_user(request, org) + courses_iter, in_process_course_actions = get_courses_accessible_to_user(request) split_archived = settings.FEATURES.get('ENABLE_SEPARATE_ARCHIVED_COURSES', False) active_courses, archived_courses = _process_courses_list(courses_iter, in_process_course_actions, split_archived) in_process_course_actions = [format_in_process_course_view(uca) for uca in in_process_course_actions] @@ -1637,7 +1633,6 @@ def get_course_context_v2(request): # 'cms.djangoapps.contentstore.utils' (most likely due to a circular import) from cms.djangoapps.contentstore.views.course import ( get_courses_accessible_to_user, - ENABLE_GLOBAL_STAFF_OPTIMIZATION, ) def format_in_process_course_view(uca): @@ -1664,10 +1659,7 @@ def format_in_process_course_view(uca): ) if uca.state == CourseRerunUIStateManager.State.FAILED else '' } - optimization_enabled = GlobalStaff().has_user(request.user) and ENABLE_GLOBAL_STAFF_OPTIMIZATION.is_enabled() - - org = request.GET.get('org', '') if optimization_enabled else None - courses_iter, in_process_course_actions = get_courses_accessible_to_user(request, org) + courses_iter, in_process_course_actions = get_courses_accessible_to_user(request) in_process_course_actions = [format_in_process_course_view(uca) for uca in in_process_course_actions] return courses_iter, in_process_course_actions @@ -1685,7 +1677,6 @@ def get_home_context(request, no_course=False): _accessible_libraries_iter, _get_course_creator_status, _format_library_for_view, - ENABLE_GLOBAL_STAFF_OPTIMIZATION, ) from cms.djangoapps.contentstore.views.library import ( user_can_view_create_library_button, @@ -1698,8 +1689,6 @@ def get_home_context(request, no_course=False): archived_courses = [] in_process_course_actions = [] - optimization_enabled = GlobalStaff().has_user(request.user) and ENABLE_GLOBAL_STAFF_OPTIMIZATION.is_enabled() - user = request.user libraries = [] @@ -1728,7 +1717,6 @@ def get_home_context(request, no_course=False): 'rerun_creator_status': GlobalStaff().has_user(user), 'allow_unicode_course_id': settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID', False), 'allow_course_reruns': settings.FEATURES.get('ALLOW_COURSE_RERUNS', True), - 'optimization_enabled': optimization_enabled, 'active_tab': 'courses', 'allowed_organizations': get_allowed_organizations(user), 'allowed_organizations_for_libraries': get_allowed_organizations_for_libraries(user), diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index 244804c3062b..e05ac894d6d8 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -139,9 +139,6 @@ 'get_course_and_check_access'] WAFFLE_NAMESPACE = 'studio_home' -ENABLE_GLOBAL_STAFF_OPTIMIZATION = WaffleSwitch( # lint-amnesty, pylint: disable=toggle-missing-annotation - f'{WAFFLE_NAMESPACE}.enable_global_staff_optimization', __name__ -) class AccessListFallback(Exception): @@ -394,15 +391,12 @@ def get_in_process_course_actions(request): ] -def _accessible_courses_summary_iter(request, org=None): +def _accessible_courses_summary_iter(request): """ List all courses available to the logged in user by iterating through all the courses Arguments: request: the request object - org (string): if not None, this value will limit the courses returned. An empty - string will result in no courses, and otherwise only courses with the - specified org will be returned. The default value is None. """ def course_filter(course_summary): """ @@ -416,9 +410,7 @@ def course_filter(course_summary): enable_home_page_api_v2 = settings.FEATURES["ENABLE_HOME_PAGE_COURSE_API_V2"] - if org is not None: - courses_summary = [] if org == '' else CourseOverview.get_all_courses(orgs=[org]) - elif enable_home_page_api_v2: + if enable_home_page_api_v2: # If the new home page API is enabled, we should use the Django ORM to filter and order the courses courses_summary = CourseOverview.get_all_courses() else: @@ -765,21 +757,17 @@ def course_index(request, course_key): @function_trace('get_courses_accessible_to_user') -def get_courses_accessible_to_user(request, org=None): +def get_courses_accessible_to_user(request): """ Try to get all courses by first reversing django groups and fallback to old method if it fails Note: overhead of pymongo reads will increase if getting courses from django groups fails Arguments: request: the request object - org (string): for global staff users ONLY, this value will be used to limit - the courses returned. A value of None will have no effect (all courses - returned), an empty string will result in no courses, and otherwise only courses with the - specified org will be returned. The default value is None. """ if GlobalStaff().has_user(request.user): # user has global access so no need to get courses from django groups - courses, in_process_course_actions = _accessible_courses_summary_iter(request, org) + courses, in_process_course_actions = _accessible_courses_summary_iter(request) else: try: courses, in_process_course_actions = _accessible_courses_list_from_groups(request) From f6f72290b79de36f3c045355a5acc725c8fef3a9 Mon Sep 17 00:00:00 2001 From: Muhammad Arslan Date: Wed, 1 Jan 2025 21:14:21 +0500 Subject: [PATCH 05/18] build: Remove unused imports --- .../contentstore/rest_api/v1/views/tests/test_home.py | 4 ---- .../contentstore/rest_api/v2/views/tests/test_home.py | 1 - cms/djangoapps/contentstore/views/course.py | 1 - 3 files changed, 6 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py index e78558bba330..31cb606b5d4b 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py @@ -8,15 +8,11 @@ from django.conf import settings from django.test import override_settings from django.urls import reverse -from edx_toggles.toggles.testutils import ( - override_waffle_switch, -) from rest_framework import status from cms.djangoapps.contentstore.tests.utils import CourseTestCase from cms.djangoapps.contentstore.tests.test_libraries import LibraryTestCase from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory -from xmodule.modulestore.tests.factories import CourseFactory FEATURES_WITH_HOME_PAGE_COURSE_V2_API = settings.FEATURES.copy() diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_home.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_home.py index bf3332e1e3cd..e899019b4f17 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_home.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_home.py @@ -10,7 +10,6 @@ from django.conf import settings from django.test import override_settings from django.urls import reverse -from edx_toggles.toggles.testutils import override_waffle_switch from rest_framework import status from cms.djangoapps.contentstore.tests.utils import CourseTestCase diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index e05ac894d6d8..3b8cb7659e0e 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -23,7 +23,6 @@ from django.views.decorators.csrf import ensure_csrf_cookie from django.views.decorators.http import require_GET, require_http_methods from edx_django_utils.monitoring import function_trace -from edx_toggles.toggles import WaffleSwitch from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import BlockUsageLocator From 343d5216193ed4df539d6bd14fa59cad82821963 Mon Sep 17 00:00:00 2001 From: Muhammad Arslan Date: Thu, 2 Jan 2025 22:42:19 +0500 Subject: [PATCH 06/18] chore: Remove unused variable and rebased --- cms/djangoapps/contentstore/views/course.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index 3b8cb7659e0e..064cb1ad25e0 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -137,8 +137,6 @@ 'group_configurations_list_handler', 'group_configurations_detail_handler', 'get_course_and_check_access'] -WAFFLE_NAMESPACE = 'studio_home' - class AccessListFallback(Exception): """ From 5f302a3ae70f1fbc001536444592dc9b5fc831fb Mon Sep 17 00:00:00 2001 From: Muhammad Anas Date: Fri, 10 Jan 2025 18:53:04 +0500 Subject: [PATCH 07/18] fix: date label for instructor paced courses --- lms/djangoapps/courseware/date_summary.py | 2 +- .../courseware/tests/test_date_summary.py | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/courseware/date_summary.py b/lms/djangoapps/courseware/date_summary.py index 4123da5f38b9..e6bd5ef70597 100644 --- a/lms/djangoapps/courseware/date_summary.py +++ b/lms/djangoapps/courseware/date_summary.py @@ -268,7 +268,7 @@ def date_type(self): @property def title(self): enrollment = CourseEnrollment.get_enrollment(self.user, self.course_id) - if enrollment and self.course.end and enrollment.created > self.course.end: + if self.course.self_paced and enrollment and self.course.start and enrollment.created > self.course.start: return gettext_lazy('Enrollment Date') return gettext_lazy('Course starts') diff --git a/lms/djangoapps/courseware/tests/test_date_summary.py b/lms/djangoapps/courseware/tests/test_date_summary.py index 27e7f1a3c226..e487af8a1f70 100644 --- a/lms/djangoapps/courseware/tests/test_date_summary.py +++ b/lms/djangoapps/courseware/tests/test_date_summary.py @@ -400,6 +400,37 @@ def test_course_start_date(self): block = CourseStartDate(course, user) assert block.date == course.start + @ddt.data( + # Instructor-paced course: Use course start date + (False, datetime(2025, 1, 10, tzinfo=utc), datetime(2025, 1, 12, tzinfo=utc), + datetime(2025, 1, 10, tzinfo=utc), 'Course starts'), + + # Self-paced course: Enrollment created later than course start + (True, datetime(2025, 1, 10, tzinfo=utc), datetime(2025, 1, 12), datetime(2025, 1, 12, tzinfo=utc), + 'Enrollment Date'), + + # Self-paced course: Enrollment created earlier than course start + (True, datetime(2025, 1, 10, tzinfo=utc), datetime(2025, 1, 8), datetime(2025, 1, 10, tzinfo=utc), + 'Course starts'), + + # Self-paced course: No enrollment + (True, datetime(2025, 1, 10, tzinfo=utc), None, datetime(2025, 1, 10, tzinfo=utc), 'Course starts'), + ) + @ddt.unpack + def test_course_start_date_label(self, self_paced, course_start, enrollment_created, expected_date, expected_title): + """ + Test the CourseStartDate class has correct label for course start date + """ + course = CourseFactory(self_paced=self_paced, start=course_start) + user = create_user() + if enrollment_created: + enrollment = CourseEnrollmentFactory(course_id=course.id, user=user, mode=CourseMode.VERIFIED) + enrollment.created = enrollment_created + enrollment.save() + date_summary = CourseStartDate(user=user, course=course) + self.assertEqual(date_summary.date, expected_date) + self.assertEqual(str(date_summary.title), expected_title) + ## Tests Course End Date Block def test_course_end_date_for_certificate_eligible_mode(self): course = create_course_run(days_till_start=-1) From 511b440045d650666ae1f3bf0ba5bf47d29b376b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 14 Jan 2025 19:44:53 +0500 Subject: [PATCH 08/18] feat: Upgrade Python dependency edx-enterprise (#36105) fix: Log all learner transmission records. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` Co-authored-by: zamanafzal <11922730+zamanafzal@users.noreply.github.com> Co-authored-by: Zaman Afzal --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 79e55d633c7d..865d224beab7 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -80,7 +80,7 @@ django-storages<1.14.4 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==5.5.2 +edx-enterprise==5.6.1 # Date: 2024-05-09 # This has to be constrained as well because newer versions of edx-i18n-tools need the diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index a6f79ba3cd7d..86431ca50e3e 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -468,7 +468,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.5.2 +edx-enterprise==5.6.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index e353c37e9541..e328b6dcb1f2 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -747,7 +747,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.5.2 +edx-enterprise==5.6.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 68cebb2d3e6d..97c9de68dfd9 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -555,7 +555,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.5.2 +edx-enterprise==5.6.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index e27fb1195b95..5e99738b3cc8 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -576,7 +576,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.5.2 +edx-enterprise==5.6.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From 755bf9809127988c0278fe5aef2b7c80aa34b742 Mon Sep 17 00:00:00 2001 From: Peter Kulko <93188219+PKulkoRaccoonGang@users.noreply.github.com> Date: Tue, 14 Jan 2025 18:56:51 +0200 Subject: [PATCH 09/18] feat: [FC-0070] Listen to xBlock interaction events (#35694) This is part of the effort to support the new Studio Unit Page embedded in the authoring MFE. It introduces several changes to improve the handling of XBlock events and the user interface in the CMS. The most important changes include adding event listeners for message handling, refining the postMessage logic, and updating the CSS for better visual consistency. --- cms/static/js/views/container.js | 11 ++ cms/static/js/views/pages/container.js | 118 ++++++++++++++++-- .../sass/course-unit-mfe-iframe-bundle.scss | 6 + .../partials/cms/theme/_variables-v1.scss | 2 + 4 files changed, 124 insertions(+), 13 deletions(-) diff --git a/cms/static/js/views/container.js b/cms/static/js/views/container.js index 9370dfdc29d5..7bf3372c6148 100644 --- a/cms/static/js/views/container.js +++ b/cms/static/js/views/container.js @@ -70,6 +70,17 @@ function($, _, XBlockView, ModuleUtils, gettext, StringUtils, NotificationView) newParent = undefined; }, update: function(event, ui) { + try { + window.parent.postMessage( + { + type: 'refreshPositions', + message: 'Refresh positions of all xblocks', + payload: {} + }, document.referrer + ); + } catch (e) { + console.error(e); + } // When dragging from one ol to another, this method // will be called twice (once for each list). ui.sender will // be null if the change is related to the list the element diff --git a/cms/static/js/views/pages/container.js b/cms/static/js/views/pages/container.js index fb8fd2482d4e..304e3bc92ddf 100644 --- a/cms/static/js/views/pages/container.js +++ b/cms/static/js/views/pages/container.js @@ -391,12 +391,16 @@ function($, _, Backbone, gettext, BasePage, editXBlock: function(event, options) { event.preventDefault(); + const isAccessButton = event.currentTarget.className === 'access-button'; + const primaryHeader = $(event.target).closest('.xblock-header-primary, .nav-actions'); + const usageId = encodeURI(primaryHeader.attr('data-usage-id')); try { - if (this.options.isIframeEmbed) { - window.parent.postMessage( + if (this.options.isIframeEmbed && isAccessButton) { + return window.parent.postMessage( { - type: 'editXBlock', - payload: {} + type: 'manageXBlockAccess', + message: 'Open the manage access modal', + payload: { usageId } }, document.referrer ); } @@ -417,8 +421,26 @@ function($, _, Backbone, gettext, BasePage, || (useNewProblemEditor === 'True' && blockType === 'problem') ) { var destinationUrl = primaryHeader.attr('authoring_MFE_base_url') - + '/' + blockType - + '/' + encodeURI(primaryHeader.attr('data-usage-id')); + + '/' + blockType + + '/' + encodeURI(primaryHeader.attr('data-usage-id')); + + try { + if (this.options.isIframeEmbed) { + return window.parent.postMessage( + { + type: 'newXBlockEditor', + message: 'Open the new XBlock editor', + payload: { + blockType, + usageId: encodeURI(primaryHeader.attr('data-usage-id')), + } + }, document.referrer + ); + } + } catch (e) { + console.error(e); + } + var upstreamRef = primaryHeader.attr('data-upstream-ref'); if(upstreamRef) { destinationUrl += '?upstreamLibRef=' + upstreamRef; @@ -548,6 +570,65 @@ function($, _, Backbone, gettext, BasePage, // Code in 'base.js' normally handles toggling these dropdowns but since this one is // not present yet during the domReady event, we have to handle displaying it ourselves. subMenu.classList.toggle('is-shown'); + + if (!subMenu.classList.contains('is-shown') && this.options.isIframeEmbed) { + try { + window.parent.postMessage( + { + type: 'toggleCourseXBlockDropdown', + message: 'Adjust the height of the dropdown menu', + payload: { courseXBlockDropdownHeight: 0 } + }, document.referrer + ); + } catch (error) { + console.error(error); + } + } + + // Calculate the viewport height and the dropdown menu height. + // Check if the dropdown would overflow beyond the iframe height based on the user's click position. + // If the dropdown overflows, adjust its position to display above the click point. + const courseUnitXBlockIframeHeight = window.innerHeight; + const courseXBlockDropdownHeight = subMenu.offsetHeight; + const clickYPosition = event.clientY; + + if (courseUnitXBlockIframeHeight < courseXBlockDropdownHeight) { + // If the dropdown menu is taller than the iframe, adjust the height of the dropdown menu. + try { + window.parent.postMessage( + { + type: 'toggleCourseXBlockDropdown', + message: 'Adjust the height of the dropdown menu', + payload: { courseXBlockDropdownHeight }, + }, document.referrer + ); + } catch (error) { + console.error(error); + } + } else if ((courseXBlockDropdownHeight + clickYPosition) > courseUnitXBlockIframeHeight) { + if (courseXBlockDropdownHeight > courseUnitXBlockIframeHeight / 2) { + // If the dropdown menu is taller than half the iframe, send a message to adjust its height. + try { + window.parent.postMessage( + { + type: 'toggleCourseXBlockDropdown', + message: 'Adjust the height of the dropdown menu', + payload: { + courseXBlockDropdownHeight: courseXBlockDropdownHeight / 2, + }, + }, document.referrer + ); + } catch (error) { + console.error(error); + } + } else { + // Move the dropdown menu upward to prevent it from overflowing out of the viewport. + if (this.options.isIframeEmbed) { + subMenu.style.top = `-${courseXBlockDropdownHeight}px`; + } + } + } + // if propagation is not stopped, the event will bubble up to the // body element, which will close the dropdown. event.stopPropagation(); @@ -588,12 +669,15 @@ function($, _, Backbone, gettext, BasePage, copyXBlock: function(event) { event.preventDefault(); + const primaryHeader = $(event.target).closest('.xblock-header-primary, .nav-actions'); + const usageId = encodeURI(primaryHeader.attr('data-usage-id')); try { if (this.options.isIframeEmbed) { - window.parent.postMessage( + return window.parent.postMessage( { type: 'copyXBlock', - payload: {} + message: 'Copy the XBlock', + payload: { usageId } }, document.referrer ); } @@ -645,12 +729,16 @@ function($, _, Backbone, gettext, BasePage, duplicateXBlock: function(event) { event.preventDefault(); + const primaryHeader = $(event.target).closest('.xblock-header-primary, .nav-actions'); + const blockType = primaryHeader.attr('data-block-type'); + const usageId = encodeURI(primaryHeader.attr('data-usage-id')); try { if (this.options.isIframeEmbed) { - window.parent.postMessage( + return window.parent.postMessage( { type: 'duplicateXBlock', - payload: {} + message: 'Duplicate the XBlock', + payload: { blockType, usageId } }, document.referrer ); } @@ -702,12 +790,15 @@ function($, _, Backbone, gettext, BasePage, deleteXBlock: function(event) { event.preventDefault(); + const primaryHeader = $(event.target).closest('.xblock-header-primary, .nav-actions'); + const usageId = encodeURI(primaryHeader.attr('data-usage-id')); try { if (this.options.isIframeEmbed) { - window.parent.postMessage( + return window.parent.postMessage( { type: 'deleteXBlock', - payload: {} + message: 'Delete the XBlock', + payload: { usageId } }, document.referrer ); } @@ -868,12 +959,13 @@ function($, _, Backbone, gettext, BasePage, || (useNewProblemEditor === 'True' && blockType.includes('problem'))) ){ var destinationUrl; - if (useVideoGalleryFlow === "True" && blockType.includes("video")) { + if (useVideoGalleryFlow === 'True' && blockType.includes('video')) { destinationUrl = this.$('.xblock-header-primary').attr("authoring_MFE_base_url") + '/course-videos/' + encodeURI(data.locator); } else { destinationUrl = this.$('.xblock-header-primary').attr("authoring_MFE_base_url") + '/' + blockType[1] + '/' + encodeURI(data.locator); } + window.location.href = destinationUrl; return; } diff --git a/cms/static/sass/course-unit-mfe-iframe-bundle.scss b/cms/static/sass/course-unit-mfe-iframe-bundle.scss index a71882f1c355..7176300da114 100644 --- a/cms/static/sass/course-unit-mfe-iframe-bundle.scss +++ b/cms/static/sass/course-unit-mfe-iframe-bundle.scss @@ -53,6 +53,7 @@ background-color: $primary; color: $white; border-color: $transparent; + color: $white; } &:focus { @@ -327,6 +328,7 @@ .wrapper-content.wrapper { padding: $baseline / 4; + background-color: #f8f7f6; } .btn-default.action-edit.title-edit-button { @@ -656,3 +658,7 @@ select { .wrapper-comp-setting.metadata-list-enum .action.setting-clear.active { margin-top: 0; } + +.wrapper-xblock .xblock-header-primary .header-actions .wrapper-nav-sub { + z-index: $zindex-dropdown; +} diff --git a/cms/static/sass/partials/cms/theme/_variables-v1.scss b/cms/static/sass/partials/cms/theme/_variables-v1.scss index a008210b25b2..c48d78ba8481 100644 --- a/cms/static/sass/partials/cms/theme/_variables-v1.scss +++ b/cms/static/sass/partials/cms/theme/_variables-v1.scss @@ -313,3 +313,5 @@ $light-background-color: #e1dddb !default; $border-color: #707070 !default; $base-font-size: 18px !default; $dark: #212529; + +$zindex-dropdown: 100; From 7fa68243a06f451e79d08007d17089bb5e5ad203 Mon Sep 17 00:00:00 2001 From: "Juan Carlos Iasenza (Aulasneo)" <166159387+jciasenza@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:43:51 -0300 Subject: [PATCH 10/18] style: Explore Courses button doesn't follow the same style conventions of MFE header (#35994) --- lms/templates/header/navbar-authenticated.html | 2 +- lms/templates/header/navbar-not-authenticated.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lms/templates/header/navbar-authenticated.html b/lms/templates/header/navbar-authenticated.html index 58c3924ea9a7..c9ea97a423b5 100644 --- a/lms/templates/header/navbar-authenticated.html +++ b/lms/templates/header/navbar-authenticated.html @@ -48,7 +48,7 @@
- ${_('Discover New')} + ${_("Discover New")}
% endif diff --git a/lms/templates/header/navbar-not-authenticated.html b/lms/templates/header/navbar-not-authenticated.html index 61448b73bd84..b50e535acda6 100644 --- a/lms/templates/header/navbar-not-authenticated.html +++ b/lms/templates/header/navbar-not-authenticated.html @@ -39,7 +39,7 @@ % if allows_login: % if can_discover_courses:
- ${_('Explore courses')} + ${_("Explore courses")}
%endif % endif From e859d7e9d9551426a605a1d6f21301adf29153d0 Mon Sep 17 00:00:00 2001 From: jawad khan Date: Wed, 15 Jan 2025 14:12:34 +0500 Subject: [PATCH 11/18] feat: Add course id in notifications list api for mobile Devices (#36106) --- openedx/core/djangoapps/notifications/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openedx/core/djangoapps/notifications/serializers.py b/openedx/core/djangoapps/notifications/serializers.py index 80b1577b6355..b5e22a36a682 100644 --- a/openedx/core/djangoapps/notifications/serializers.py +++ b/openedx/core/djangoapps/notifications/serializers.py @@ -202,6 +202,7 @@ class Meta: 'content_context', 'content', 'content_url', + 'course_id', 'last_read', 'last_seen', 'created', From f64a3a62c572ee45fe2362222acd46ed5210a493 Mon Sep 17 00:00:00 2001 From: irfanuddinahmad Date: Thu, 2 Jan 2025 19:17:28 +0500 Subject: [PATCH 12/18] feat: Disabled MITx from program nudge emails --- .../commands/send_program_course_nudge_email.py | 5 ++++- lms/envs/common.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/program_enrollments/management/commands/send_program_course_nudge_email.py b/lms/djangoapps/program_enrollments/management/commands/send_program_course_nudge_email.py index c095b2d161a1..af2bfdca752d 100644 --- a/lms/djangoapps/program_enrollments/management/commands/send_program_course_nudge_email.py +++ b/lms/djangoapps/program_enrollments/management/commands/send_program_course_nudge_email.py @@ -15,6 +15,7 @@ from django.contrib.sites.models import Site from django.core.management import BaseCommand from django.utils import timezone +from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import CourseLocator from common.djangoapps.track import segment @@ -140,7 +141,9 @@ def get_course_run_to_suggest(self, candidate_programs, completed_course_id, use ) break for course_run in candidate_course['course_runs']: - if self.valid_course_run(course_run) and course_run['key'] != completed_course_id: + course_org = CourseKey.from_string(course_run['key']).org + if self.valid_course_run(course_run) and course_run['key'] != completed_course_id \ + and course_org not in settings.DISABLED_ORGS_FOR_PROGRAM_NUDGE: return program, course_run, candidate_course return None, None, None diff --git a/lms/envs/common.py b/lms/envs/common.py index e354f75a8530..cb7643c3668e 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -5376,6 +5376,14 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring NOTIFICATION_TYPE_ICONS = {} DEFAULT_NOTIFICATION_ICON_URL = "" +############## NUDGE EMAILS ############### +# .. setting_name: DISABLED_ORGS_FOR_PROGRAM_NUDGE +# .. setting_default: [] +# .. setting_description: List of organization codes that should be disabled +# .. for program nudge emails. +# .. eg ['BTDx', 'MYTx'] +DISABLED_ORGS_FOR_PROGRAM_NUDGE = [] + ############################ AI_TRANSLATIONS ################################## AI_TRANSLATIONS_API_URL = 'http://localhost:18760/api/v1' From 749b3ebc82245e2d34a5eb9a5f13bc4939c2997d Mon Sep 17 00:00:00 2001 From: Brian Mesick <112640379+bmtcril@users.noreply.github.com> Date: Wed, 15 Jan 2025 10:05:07 -0500 Subject: [PATCH 13/18] build: Cache images in unit tests (#36110) Another attempt to stop Dockerhub from rate limiting us in CI. If this works here, we'll try to add this caching to other relevant workflows. https://github.com/openedx/axim-engineering/issues/1350#issuecomment-2591237325 --- .github/workflows/unit-tests.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index d399d38770b9..9867ac72f273 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -73,15 +73,13 @@ jobs: run: | sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev lynx - # Try to log into DockerHub so that we're less likely to be rate-limited when pulling certain images. - # This will fail on any edx-platform fork which doesn't explicitly define its own DockerHub creds. - # That's OK--if we fail to log in, we'll proceed anonymously, and hope we don't get rate-limited. - - name: Try to log into Docker Hub - uses: docker/login-action@v3.3.0 - continue-on-error: true + # We pull this image a lot, and Dockerhub will rate limit us if we pull too often. + # This is an attempt to cache the image for better performance and to work around that. + # It will cache all pulled images, so if we add new images to this we'll need to update the key. + - name: Cache Docker images + uses: ScribeMD/docker-cache@0.5.0 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + key: docker-${{ runner.os }}-mongo-${{ matrix.mongo-version }} - name: Start MongoDB uses: supercharge/mongodb-github-action@1.11.0 From b96a3bf249ecc104b4882e67738d06499c1a0245 Mon Sep 17 00:00:00 2001 From: Jillian Date: Thu, 16 Jan 2025 04:02:37 +1030 Subject: [PATCH 14/18] fix: allow_to_create_new_org checks org autocreate [FC-0076] (#36094) Updates the StudioHome API's allow_to_create_new_org to require both organization-creation permissions and ORGANIZATION_AUTOCREATE to be enabled. It also adds the list of "allowed organizations for libraries" to the Studio Home API so that the Authoring MFE can use it. --- .../rest_api/v1/serializers/home.py | 4 ++ .../contentstore/rest_api/v1/views/home.py | 9 ++- .../rest_api/v1/views/tests/test_home.py | 14 ++++- .../tests/test_content_libraries.py | 58 +++++++++++++++++++ .../djangoapps/content_libraries/views.py | 12 ++++ 5 files changed, 95 insertions(+), 2 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py b/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py index 36166e42a5a6..fdc06e9291d0 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py @@ -50,6 +50,10 @@ class StudioHomeSerializer(serializers.Serializer): child=serializers.CharField(), allow_empty=True ) + allowed_organizations_for_libraries = serializers.ListSerializer( + child=serializers.CharField(), + allow_empty=True + ) archived_courses = CourseCommonSerializer(required=False, many=True) can_access_advanced_settings = serializers.BooleanField() can_create_organizations = serializers.BooleanField() diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/home.py b/cms/djangoapps/contentstore/rest_api/v1/views/home.py index 2d5360d38d6c..62b56533878f 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/home.py @@ -5,6 +5,7 @@ from rest_framework.request import Request from rest_framework.response import Response from rest_framework.views import APIView +from organizations import api as org_api from openedx.core.lib.api.view_utils import view_auth_classes from ....utils import get_home_context, get_course_context, get_library_context @@ -51,6 +52,7 @@ def get(self, request: Request): "allow_to_create_new_org": true, "allow_unicode_course_id": false, "allowed_organizations": [], + "allowed_organizations_for_libraries": [], "archived_courses": [], "can_access_advanced_settings": true, "can_create_organizations": true, @@ -79,7 +81,12 @@ def get(self, request: Request): home_context = get_home_context(request, True) home_context.update({ - 'allow_to_create_new_org': settings.FEATURES.get('ENABLE_CREATOR_GROUP', True) and request.user.is_staff, + # 'allow_to_create_new_org' is actually about auto-creating organizations + # (e.g. when creating a course or library), so we add an additional test. + 'allow_to_create_new_org': ( + home_context['can_create_organizations'] and + org_api.is_autocreate_enabled() + ), 'studio_name': settings.STUDIO_NAME, 'studio_short_name': settings.STUDIO_SHORT_NAME, 'studio_request_email': settings.FEATURES.get('STUDIO_REQUEST_EMAIL', ''), diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py index 31cb606b5d4b..8fe246cf23fd 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py @@ -32,9 +32,10 @@ def setUp(self): self.url = reverse("cms.djangoapps.contentstore:v1:home") self.expected_response = { "allow_course_reruns": True, - "allow_to_create_new_org": False, + "allow_to_create_new_org": True, "allow_unicode_course_id": False, "allowed_organizations": [], + "allowed_organizations_for_libraries": [], "archived_courses": [], "can_access_advanced_settings": True, "can_create_organizations": True, @@ -78,6 +79,17 @@ def test_home_page_studio_with_meilisearch_enabled(self): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertDictEqual(expected_response, response.data) + @override_settings(ORGANIZATIONS_AUTOCREATE=False) + def test_home_page_studio_with_org_autocreate_disabled(self): + """Check response content when Organization autocreate is disabled""" + response = self.client.get(self.url) + + expected_response = self.expected_response + expected_response["allow_to_create_new_org"] = False + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertDictEqual(expected_response, response.data) + def test_taxonomy_list_link(self): response = self.client.get(self.url) self.assertTrue(response.data['taxonomies_enabled']) diff --git a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py index 83b277604071..f3a111cb0561 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py @@ -8,6 +8,7 @@ import ddt from django.contrib.auth.models import Group +from django.test import override_settings from django.test.client import Client from freezegun import freeze_time from opaque_keys.edx.locator import LibraryLocatorV2, LibraryUsageLocatorV2 @@ -139,6 +140,63 @@ def test_library_validation(self): 'slug': ['Enter a valid “slug” consisting of Unicode letters, numbers, underscores, or hyphens.'], } + def test_library_org_validation(self): + """ + Staff users can create libraries in any existing or auto-created organization. + """ + assert Organization.objects.filter(short_name='auto-created-org').count() == 0 + self._create_library(slug="auto-created-org-1", title="Library in an auto-created org", org='auto-created-org') + assert Organization.objects.filter(short_name='auto-created-org').count() == 1 + self._create_library(slug="existing-org-1", title="Library in an existing org", org="CL-TEST") + + @patch( + "openedx.core.djangoapps.content_libraries.views.user_can_create_organizations", + ) + @patch( + "openedx.core.djangoapps.content_libraries.views.get_allowed_organizations_for_libraries", + ) + @override_settings(ORGANIZATIONS_AUTOCREATE=False) + def test_library_org_no_autocreate(self, mock_get_allowed_organizations, mock_can_create_organizations): + """ + When org auto-creation is disabled, user must use one of their allowed orgs. + """ + mock_can_create_organizations.return_value = False + mock_get_allowed_organizations.return_value = ["CL-TEST"] + assert Organization.objects.filter(short_name='auto-created-org').count() == 0 + response = self._create_library( + slug="auto-created-org-2", + org="auto-created-org", + title="Library in an auto-created org", + expect_response=400, + ) + assert response == { + 'org': "No such organization 'auto-created-org' found.", + } + + Organization.objects.get_or_create( + short_name="not-allowed-org", + defaults={"name": "Content Libraries Test Org Membership"}, + ) + response = self._create_library( + slug="not-allowed-org", + org="not-allowed-org", + title="Library in an not-allowed org", + expect_response=400, + ) + assert response == { + 'org': "User not allowed to create libraries in 'not-allowed-org'.", + } + assert mock_can_create_organizations.call_count == 1 + assert mock_get_allowed_organizations.call_count == 1 + + self._create_library( + slug="allowed-org-2", + org="CL-TEST", + title="Library in an allowed org", + ) + assert mock_can_create_organizations.call_count == 2 + assert mock_get_allowed_organizations.call_count == 2 + @skip("This endpoint shouldn't support num_blocks and has_unpublished_*.") @patch("openedx.core.djangoapps.content_libraries.views.LibraryRootView.pagination_class.page_size", new=2) def test_list_library(self): diff --git a/openedx/core/djangoapps/content_libraries/views.py b/openedx/core/djangoapps/content_libraries/views.py index 4c14651c7961..048226c5b16c 100644 --- a/openedx/core/djangoapps/content_libraries/views.py +++ b/openedx/core/djangoapps/content_libraries/views.py @@ -99,6 +99,10 @@ from rest_framework.views import APIView from rest_framework.viewsets import GenericViewSet +from cms.djangoapps.contentstore.views.course import ( + get_allowed_organizations_for_libraries, + user_can_create_organizations, +) from openedx.core.djangoapps.content_libraries import api, permissions from openedx.core.djangoapps.content_libraries.serializers import ( ContentLibraryBlockImportTaskCreateSerializer, @@ -269,6 +273,14 @@ def post(self, request): raise ValidationError( # lint-amnesty, pylint: disable=raise-missing-from detail={"org": f"No such organization '{org_name}' found."} ) + # Ensure the user is allowed to create libraries under this org + if not ( + user_can_create_organizations(request.user) or + org_name in get_allowed_organizations_for_libraries(request.user) + ): + raise ValidationError( # lint-amnesty, pylint: disable=raise-missing-from + detail={"org": f"User not allowed to create libraries in '{org_name}'."} + ) org = Organization.objects.get(short_name=org_name) try: From 343a4cad56e10df81feb49c25372625ec534c33c Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Wed, 15 Jan 2025 17:35:13 -0500 Subject: [PATCH 15/18] docs: update settings comment --- lms/envs/production.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lms/envs/production.py b/lms/envs/production.py index 6dc6be634178..addcea0e6028 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -185,7 +185,6 @@ def get_env_setting(setting): CMS_BASE = ENV_TOKENS.get('CMS_BASE', 'studio.edx.org') ALLOWED_HOSTS = [ - # TODO: bbeggs remove this before prod, temp fix to get load testing running "*", ENV_TOKENS.get('LMS_BASE'), FEATURES['PREVIEW_LMS_BASE'], From 829f6dbf3310586ce46a6e2d8bf1c5e9647ce0aa Mon Sep 17 00:00:00 2001 From: Hunia Fatima Date: Thu, 16 Jan 2025 17:46:57 +0500 Subject: [PATCH 16/18] chore: remove dockerfile setup (#35731) * chore: remove dockerfile setup --- .dockerignore | 152 ------------------ .github/workflows/publish-ci-docker-image.yml | 0 2 files changed, 152 deletions(-) delete mode 100644 .dockerignore create mode 100644 .github/workflows/publish-ci-docker-image.yml diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index c3873d33a595..000000000000 --- a/.dockerignore +++ /dev/null @@ -1,152 +0,0 @@ -# .dockerignore for edx-platform. -# There's a lot here, please try to keep it organized. - -### Files that are not needed in the docker file - -/test_root/ -.git - -### Files private to developers - -# Files that should be git-ignored, but are hand-edited or otherwise valued, -# and so should not be destroyed by "make clean". -# start-noclean -requirements/private.txt -requirements/edx/private.in -requirements/edx/private.txt -lms/envs/private.py -cms/envs/private.py -# end-noclean - -### Python artifacts -**/*.pyc - -### Editor and IDE artifacts -**/*~ -**/*.swp -**/*.orig -**/nbproject -**/.idea/ -**/.redcar/ -**/codekit-config.json -**/.pycharm_helpers/ -**/_mac/* -**/IntelliLang.xml -**/conda_packages.xml -**/databaseSettings.xml -**/diff.xml -**/debugger.xml -**/editor.xml -**/ide.general.xml -**/inspection/Default.xml -**/other.xml -**/packages.xml -**/web-browsers.xml - -### NFS artifacts -**/.nfs* - -### OS X artifacts -**/*.DS_Store -**/.AppleDouble -**/:2e_* -**/:2e# - -### Internationalization artifacts -**/*.mo -**/*.po -**/*.prob -**/*.dup -!**/django.po -!**/django.mo -!**/djangojs.po -!**/djangojs.mo -conf/locale/en/LC_MESSAGES/*.mo -conf/locale/fake*/LC_MESSAGES/*.po -conf/locale/fake*/LC_MESSAGES/*.mo - -### Testing artifacts -**/.testids/ -**/.noseids -**/nosetests.xml -**/.cache/ -**/.coverage -**/.coverage.* -**/coverage.xml -**/cover/ -**/cover_html/ -**/reports/ -**/jscover.log -**/jscover.log.* -**/.pytest_cache/ -**/pytest_task*.txt -**/.tddium* -common/test/data/test_unicode/static/ -test_root/courses/ -test_root/data/test_bare.git/ -test_root/export_course_repos/ -test_root/paver_logs/ -test_root/uploads/ -**/django-pyfs -**/.tox/ -common/test/data/badges/*.png - -### Installation artifacts -**/*.egg-info -**/.pip_download_cache/ -**/.prereqs_cache -**/.vagrant/ -**/node_modules -**/bin/ - -### Static assets pipeline artifacts -**/*.scssc -lms/static/css/ -lms/static/certificates/css/ -cms/static/css/ -common/static/common/js/vendor/ -common/static/common/css/vendor/ -common/static/bundles -**/webpack-stats.json - -### Styling generated from templates -lms/static/sass/*.css -lms/static/sass/*.css.map -lms/static/certificates/sass/*.css -lms/static/themed_sass/ -cms/static/css/ -cms/static/sass/*.css -cms/static/sass/*.css.map -cms/static/themed_sass/ -themes/**/css - -### Logging artifacts -**/log/ -**/logs -**/chromedriver.log -**/ghostdriver.log - -### Celery artifacts ### -**/celerybeat-schedule - -### Unknown artifacts -**/database.sqlite -**/courseware/static/js/mathjax/* -**/flushdb.sh -**/build -/src/ -\#*\# -**/.env/ -openedx/core/djangoapps/django_comment_common/comment_client/python -**/autodeploy.properties -**/.ws_migrations_complete -**/dist -**/*.bak - -# Visual Studio Code -**/.vscode - -# Locally generated PII reports -**/pii_report - -/Dockerfile diff --git a/.github/workflows/publish-ci-docker-image.yml b/.github/workflows/publish-ci-docker-image.yml new file mode 100644 index 000000000000..e69de29bb2d1 From b885618ae1b199f986e9061bd91c0ac3570e81bb Mon Sep 17 00:00:00 2001 From: Alison Langston <46360176+alangsto@users.noreply.github.com> Date: Thu, 16 Jan 2025 09:41:46 -0500 Subject: [PATCH 17/18] fix: remove lti-consumer-xblock pin and upgrade package (#36119) * fix: remove lti-consumer-xblock pin and upgrade package * feat: Upgrade Python dependency lti-consumer-xblock (#36120) 9.13.1 contains a fix for broken LTI 1.3 launches Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` Co-authored-by: alangsto <46360176+alangsto@users.noreply.github.com> --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- requirements/constraints.txt | 5 ----- requirements/edx/base.txt | 7 +++---- requirements/edx/development.txt | 4 ++-- requirements/edx/doc.txt | 7 +++---- requirements/edx/testing.txt | 7 +++---- 5 files changed, 11 insertions(+), 19 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 865d224beab7..f99c41c6947b 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -184,8 +184,3 @@ social-auth-app-django<=5.4.1 # We are pinning this until we can upgrade to a version of elasticsearch that uses a more recent version of urllib3. # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35126 elasticsearch==7.9.1 - -# Date 2025-01-10 -# Cause: https://github.com/openedx/edx-platform/issues/36095 -# Issue for unpinning https://github.com/openedx/edx-platform/issues/36096 -lti-consumer-xblock==9.12.1 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 86431ca50e3e..d202a8be1f60 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -705,10 +705,8 @@ lazy==1.6 # xblock loremipsum==1.0.5 # via ora2 -lti-consumer-xblock==9.12.1 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/kernel.in +lti-consumer-xblock==9.13.1 + # via -r requirements/edx/kernel.in lxml[html-clean,html_clean]==5.3.0 # via # -r requirements/edx/kernel.in @@ -944,6 +942,7 @@ pyjwt[crypto]==2.10.1 # edx-proctoring # edx-rest-api-client # firebase-admin + # lti-consumer-xblock # pylti1p3 # snowflake-connector-python # social-auth-core diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index e328b6dcb1f2..bd1d6b02bd8d 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1189,9 +1189,8 @@ loremipsum==1.0.5 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # ora2 -lti-consumer-xblock==9.12.1 +lti-consumer-xblock==9.13.1 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt lxml[html-clean]==5.3.0 @@ -1611,6 +1610,7 @@ pyjwt[crypto]==2.10.1 # edx-proctoring # edx-rest-api-client # firebase-admin + # lti-consumer-xblock # pylti1p3 # snowflake-connector-python # social-auth-core diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 97c9de68dfd9..52a4c067c0de 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -860,10 +860,8 @@ loremipsum==1.0.5 # via # -r requirements/edx/base.txt # ora2 -lti-consumer-xblock==9.12.1 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/base.txt +lti-consumer-xblock==9.13.1 + # via -r requirements/edx/base.txt lxml[html-clean]==5.3.0 # via # -r requirements/edx/base.txt @@ -1160,6 +1158,7 @@ pyjwt[crypto]==2.10.1 # edx-proctoring # edx-rest-api-client # firebase-admin + # lti-consumer-xblock # pylti1p3 # snowflake-connector-python # social-auth-core diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 5e99738b3cc8..672c51c48ae7 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -903,10 +903,8 @@ loremipsum==1.0.5 # via # -r requirements/edx/base.txt # ora2 -lti-consumer-xblock==9.12.1 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/base.txt +lti-consumer-xblock==9.13.1 + # via -r requirements/edx/base.txt lxml[html-clean]==5.3.0 # via # -r requirements/edx/base.txt @@ -1224,6 +1222,7 @@ pyjwt[crypto]==2.10.1 # edx-proctoring # edx-rest-api-client # firebase-admin + # lti-consumer-xblock # pylti1p3 # snowflake-connector-python # social-auth-core From 8aeaaf4e216b66ae0d11df5d8d677ddce62ac44c Mon Sep 17 00:00:00 2001 From: Ihor Romaniuk Date: Thu, 16 Jan 2025 18:05:26 +0100 Subject: [PATCH 18/18] feat: [FC-0070] add message events and styles to the library content page (#35785) This introduces improvements for XBlock interactions within iframes: * Add default styles for Library Content that renders in the iframe in the new Studio unit page * When the `isIframeEmbed` option is enabled, the XBlock sends a `postMessage` to the parent window. When sending such a message, the standard link transition is cancelled and the transition is carried out in the MFE. --- .../contentstore/views/tests/test_block.py | 7 +- cms/static/js/views/pages/container.js | 25 ++++- .../sass/course-unit-mfe-iframe-bundle.scss | 98 +++++++++++++++---- .../partials/cms/theme/_variables-v1.scss | 5 + cms/templates/studio_xblock_wrapper.html | 2 +- 5 files changed, 111 insertions(+), 26 deletions(-) diff --git a/cms/djangoapps/contentstore/views/tests/test_block.py b/cms/djangoapps/contentstore/views/tests/test_block.py index 1190eb239518..cfbbcac5cde5 100644 --- a/cms/djangoapps/contentstore/views/tests/test_block.py +++ b/cms/djangoapps/contentstore/views/tests/test_block.py @@ -265,9 +265,10 @@ def test_get_container_nested_container_fragment(self): html, # The instance of the wrapper class will have an auto-generated ID. Allow any # characters after wrapper. - '"/container/{}" class="action-button">\\s*View'.format( - re.escape(str(wrapper_usage_key)) - ), + ( + '"/container/{}" class="action-button xblock-view-action-button">' + '\\s*View' + ).format(re.escape(str(wrapper_usage_key))), ) @patch("cms.djangoapps.contentstore.xblock_storage_handlers.xblock_helpers.get_object_tag_counts") diff --git a/cms/static/js/views/pages/container.js b/cms/static/js/views/pages/container.js index 304e3bc92ddf..7f5e2c257e22 100644 --- a/cms/static/js/views/pages/container.js +++ b/cms/static/js/views/pages/container.js @@ -39,6 +39,7 @@ function($, _, Backbone, gettext, BasePage, 'click .manage-tags-button': 'openManageTags', 'change .header-library-checkbox': 'toggleLibraryComponent', 'click .collapse-button': 'collapseXBlock', + 'click .xblock-view-action-button': 'viewXBlockContent', }, options: { @@ -808,10 +809,6 @@ function($, _, Backbone, gettext, BasePage, this.deleteComponent(this.findXBlockElement(event.target)); }, - createPlaceholderElement: function() { - return $('
', {class: 'studio-xblock-wrapper'}); - }, - createComponent: function(template, target) { // A placeholder element is created in the correct location for the new xblock // and then onNewXBlock will replace it with a rendering of the xblock. Note that @@ -905,6 +902,26 @@ function($, _, Backbone, gettext, BasePage, } }, + viewXBlockContent: function(event) { + try { + if (this.options.isIframeEmbed) { + event.preventDefault(); + var usageId = event.currentTarget.href.split('/').pop() || ''; + window.parent.postMessage( + { + type: 'handleViewXBlockContent', + payload: { + usageId: usageId, + }, + }, document.referrer + ); + return true; + } + } catch (e) { + console.error(e); + } + }, + toggleSaveButton: function() { var $saveButton = $('.nav-actions .save-button'); if (JSON.stringify(this.selectedLibraryComponents.sort()) === JSON.stringify(this.storedSelectedLibraryComponents.sort())) { diff --git a/cms/static/sass/course-unit-mfe-iframe-bundle.scss b/cms/static/sass/course-unit-mfe-iframe-bundle.scss index 7176300da114..bc0c3901b147 100644 --- a/cms/static/sass/course-unit-mfe-iframe-bundle.scss +++ b/cms/static/sass/course-unit-mfe-iframe-bundle.scss @@ -1,14 +1,22 @@ @import 'cms/theme/variables-v1'; @import 'elements/course-unit-mfe-iframe'; +body { + min-width: 800px; +} + .wrapper { + .inner-wrapper { + max-width: 100%; + } + .wrapper-xblock { background-color: $transparent; border-radius: 6px; border: none; &:hover { - border-color: none; + border-color: transparent; } .xblock-header-primary { @@ -23,6 +31,54 @@ } } + .xblock-header-secondary { + border-radius: 0 0 4px 4px; + + .actions-list .action-item .action-button { + border-radius: 4px; + + &:hover { + background-color: $primary; + color: $white; + } + } + } + + &.level-page .xblock-message { + padding: ($baseline * .75) ($baseline * 1.2); + border-radius: 0 0 4px 4px; + + &.information { + color: $text-color; + background-color: $xblock-message-info-bg; + border-color: $xblock-message-info-border-color; + } + + &.validation.has-warnings { + color: $black; + background-color: $xblock-message-warning-bg; + border-color: $xblock-message-warning-border-color; + border-top-width: 1px; + + .icon { + color: $xblock-message-warning-border-color; + } + } + + a { + color: $primary; + } + } + + .xblock-author_view-library_content > .wrapper-xblock-message .xblock-message { + font-size: 16px; + line-height: 22px; + border-radius: 4px; + padding: ($baseline * 1.2); + box-shadow: 0 1px 2px rgba(0, 0, 0, .15), 0 1px 4px rgba(0, 0, 0, .15); + margin-bottom: ($baseline * 1.4); + } + &.level-element { box-shadow: 0 2px 4px rgba(0, 0, 0, .15), 0 2px 8px rgba(0, 0, 0, .15); margin: 0 0 ($baseline * 1.4) 0; @@ -40,28 +96,34 @@ border-bottom-right-radius: 6px; } - .wrapper-xblock .header-actions .actions-list .action-item .action-button { - @extend %button-styles; + .wrapper-xblock .header-actions .actions-list { + .action-actions-menu:last-of-type .nav-sub { + right: 120px; + } - color: $primary; + .action-item .action-button { + @extend %button-styles; - .fa-ellipsis-v { - font-size: $base-font-size; - } + color: $primary; - &:hover { - background-color: $primary; - color: $white; - border-color: $transparent; + .fa-ellipsis-v { + font-size: $base-font-size; + } + + &:hover { + background-color: $primary; + color: $white; + border-color: $transparent; color: $white; } - &:focus { - outline: 2px $transparent; - background-color: $transparent; - box-shadow: inset 0 0 0 2px $primary; - color: $primary; - border-color: $transparent; + &:focus { + outline: 2px $transparent; + background-color: $transparent; + box-shadow: inset 0 0 0 2px $primary; + color: $primary; + border-color: $transparent; + } } } @@ -631,7 +693,7 @@ select { } } -.xblock-header-primary { +.xblock-header:not(.xblock-header-library_content, .xblock-header-split_test) .xblock-header-primary { position: relative; &::before { diff --git a/cms/static/sass/partials/cms/theme/_variables-v1.scss b/cms/static/sass/partials/cms/theme/_variables-v1.scss index c48d78ba8481..0b3fe6b6e49b 100644 --- a/cms/static/sass/partials/cms/theme/_variables-v1.scss +++ b/cms/static/sass/partials/cms/theme/_variables-v1.scss @@ -315,3 +315,8 @@ $base-font-size: 18px !default; $dark: #212529; $zindex-dropdown: 100; + +$xblock-message-info-bg: #eff8fa; +$xblock-message-info-border-color: #9cd2e6; +$xblock-message-warning-bg: #fffdf0; +$xblock-message-warning-border-color: #fff6bf; diff --git a/cms/templates/studio_xblock_wrapper.html b/cms/templates/studio_xblock_wrapper.html index 41555410236a..8f4090588613 100644 --- a/cms/templates/studio_xblock_wrapper.html +++ b/cms/templates/studio_xblock_wrapper.html @@ -229,7 +229,7 @@
${_('This block contains multiple components.')}