From 6d7bc9bec39a4bb82f91ef8cbd93adf56d048382 Mon Sep 17 00:00:00 2001 From: thisiswhereitype <17139430+thisiswhereitype@users.noreply.github.com> Date: Fri, 3 May 2024 14:18:56 +0100 Subject: [PATCH 1/5] Add eol --newline option for text content in nbconvert.writers.files --- nbconvert/nbconvertapp.py | 35 +++++++++++++++++++++++++++++++---- nbconvert/writers/files.py | 20 +++++++++++++++++--- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/nbconvert/nbconvertapp.py b/nbconvert/nbconvertapp.py index cd305afbf..606433164 100755 --- a/nbconvert/nbconvertapp.py +++ b/nbconvert/nbconvertapp.py @@ -17,7 +17,16 @@ from textwrap import dedent, fill from jupyter_core.application import JupyterApp, base_aliases, base_flags -from traitlets import Bool, DottedObjectName, Instance, List, Type, Unicode, default, observe +from traitlets import ( + Bool, + DottedObjectName, + Instance, + List, + Type, + Unicode, + default, + observe, +) from traitlets.config import Configurable, catch_config_error from traitlets.utils.importstring import import_item @@ -60,6 +69,7 @@ def validate(self, obj, value): "writer": "NbConvertApp.writer_class", "post": "NbConvertApp.postprocessor_class", "output": "NbConvertApp.output_base", + "newline": "NbConvertApp.newline", "output-dir": "FilesWriter.build_directory", "reveal-prefix": "SlidesExporter.reveal_url_prefix", "nbformat": "NotebookExporter.nbformat_version", @@ -120,7 +130,10 @@ def validate(self, obj, value): ), "coalesce-streams": ( { - "NbConvertApp": {"use_output_suffix": False, "export_format": "notebook"}, + "NbConvertApp": { + "use_output_suffix": False, + "export_format": "notebook", + }, "FilesWriter": {"build_directory": ""}, "CoalesceStreamsPreprocessor": {"enabled": True}, }, @@ -304,6 +317,12 @@ def _classes_default(self): } writer_factory = Type(allow_none=True) + newline = Unicode( + None, + help="""The line ending to use when writing text. See builtin:`open`""", + allow_none=True, + ).tag(config=True) + @observe("writer_class") def _writer_class_changed(self, change): new = change["new"] @@ -523,7 +542,9 @@ def write_single_notebook(self, output, resources): if not self.writer: msg = "No writer object defined!" raise ValueError(msg) - return self.writer.write(output, resources, notebook_name=notebook_name) + return self.writer.write( + output, resources, notebook_name=notebook_name, newline=self.newline + ) def postprocess_single_notebook(self, write_results): """Step 4: Post-process the written file @@ -628,7 +649,13 @@ def document_config_options(self): """ categories = { category: [c for c in self._classes_inc_parents() if category in c.__name__.lower()] - for category in ["app", "exporter", "writer", "preprocessor", "postprocessor"] + for category in [ + "app", + "exporter", + "writer", + "preprocessor", + "postprocessor", + ] } accounted_for = {c for category in categories.values() for c in category} categories["other"] = [c for c in self._classes_inc_parents() if c not in accounted_for] diff --git a/nbconvert/writers/files.py b/nbconvert/writers/files.py index 8349360a4..dc7b2ca7c 100644 --- a/nbconvert/writers/files.py +++ b/nbconvert/writers/files.py @@ -77,7 +77,7 @@ def _write_items(self, items, build_dir): with open(dest, "wb") as f: f.write(data) - def write(self, output, resources, notebook_name=None, **kw): + def write(self, output, resources, notebook_name=None, newline=None, **kw): """ Consume and write Jinja output to the file system. Output directory is set via the 'build_directory' variable of this instance (a @@ -147,11 +147,25 @@ def write(self, output, resources, notebook_name=None, **kw): dest_path = Path(build_directory) / dest # Write conversion results. - self.log.info("Writing %i bytes to %s", len(output), dest_path) + if "\\\\" in repr(newline): + self.log.warning( + "Argument 'newline' contains escaped escape characters: %s", + format(repr(newline)), + ) + if isinstance(output, str): - with open(dest_path, "w", encoding="utf-8") as f: + self.log.info( + "Writing %i len string to %s, (repr(newline): %s)", + len(output), + dest_path, + repr(newline), + ) + with open(dest_path, "w", encoding="utf-8", newline=newline) as f: f.write(output) + else: + # newline only applies in text mode. + self.log.info("Writing %i bytes to %s", len(output), dest_path) with open(dest_path, "wb") as f: f.write(output) From b6952b05c009a465328595c1c3bebf3bc80125bd Mon Sep 17 00:00:00 2001 From: Joe Scott <17139430+thisiswhereitype@users.noreply.github.com> Date: Wed, 15 May 2024 20:42:20 +0100 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6109e5728..c42f0a1f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1586,6 +1586,7 @@ raw template {%- endblock in_prompt -%} """ + exporter_attr = AttrExporter() output_attr, _ = exporter_attr.from_notebook_node(nb) assert "raw template" in output_attr From 102f40e4688552e3398dbbc788717758547c8ae4 Mon Sep 17 00:00:00 2001 From: Joe Scott <17139430+thisiswhereitype@users.noreply.github.com> Date: Wed, 15 May 2024 20:42:20 +0100 Subject: [PATCH 3/5] Add options --use-crlf-newline and --use-lf-newline --- nbconvert/nbconvertapp.py | 55 +++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/nbconvert/nbconvertapp.py b/nbconvert/nbconvertapp.py index 606433164..a1186f681 100755 --- a/nbconvert/nbconvertapp.py +++ b/nbconvert/nbconvertapp.py @@ -199,6 +199,11 @@ def validate(self, obj, value): }, """Whether the HTML in Markdown cells and cell outputs should be sanitized..""", ), + "use-crlf-newline": ( + {"NbConvertApp": {"newline": "\r\n"}}, + "Implies CLRF newline.", + ), + "use-lf-newline": ({"NbConvertApp": {"newline": "\n"}}, "Implies LF newline."), } ) @@ -317,12 +322,6 @@ def _classes_default(self): } writer_factory = Type(allow_none=True) - newline = Unicode( - None, - help="""The line ending to use when writing text. See builtin:`open`""", - allow_none=True, - ).tag(config=True) - @observe("writer_class") def _writer_class_changed(self, change): new = change["new"] @@ -330,6 +329,12 @@ def _writer_class_changed(self, change): new = self.writer_aliases[new.lower()] self.writer_factory = import_item(new) + newline = Unicode( + None, + help="""The line ending to use when writing text, defaults to builtin:`open`""", + allow_none=True, + ).tag(config=True) + # Post-processor specific variables postprocessor = Instance( "nbconvert.postprocessors.base.PostProcessorBase", @@ -342,7 +347,9 @@ def _writer_class_changed(self, change): help="""PostProcessor class used to write the results of the conversion""" ).tag(config=True) - postprocessor_aliases = {"serve": "nbconvert.postprocessors.serve.ServePostProcessor"} + postprocessor_aliases = { + "serve": "nbconvert.postprocessors.serve.ServePostProcessor" + } postprocessor_factory = Type(None, allow_none=True) @observe("postprocessor_class") @@ -410,7 +417,9 @@ def init_notebooks(self): # Use glob to find matching filenames. Allow the user to convert # notebooks without having to type the extension. globbed_files = glob.glob(pattern, recursive=self.recursive_glob) - globbed_files.extend(glob.glob(pattern + ".ipynb", recursive=self.recursive_glob)) + globbed_files.extend( + glob.glob(pattern + ".ipynb", recursive=self.recursive_glob) + ) if not globbed_files: self.log.warning("pattern %r matched no files", pattern) @@ -424,7 +433,10 @@ def init_writer(self): self._writer_class_changed({"new": self.writer_class}) if self.writer_factory: self.writer = self.writer_factory(parent=self) - if hasattr(self.writer, "build_directory") and self.writer.build_directory != "": + if ( + hasattr(self.writer, "build_directory") + and self.writer.build_directory != "" + ): self.use_output_suffix = False def init_postprocessor(self): @@ -501,13 +513,17 @@ def export_single_notebook(self, notebook_filename, resources, input_buffer=None """ try: if input_buffer is not None: - output, resources = self.exporter.from_file(input_buffer, resources=resources) + output, resources = self.exporter.from_file( + input_buffer, resources=resources + ) else: output, resources = self.exporter.from_filename( notebook_filename, resources=resources ) except ConversionException: - self.log.error("Error while converting '%s'", notebook_filename, exc_info=True) # noqa: G201 + self.log.error( + "Error while converting '%s'", notebook_filename, exc_info=True + ) # noqa: G201 self.exit(1) return output, resources @@ -576,7 +592,9 @@ def convert_single_notebook(self, notebook_filename, input_buffer=None): argument. """ if input_buffer is None: - self.log.info("Converting notebook %s to %s", notebook_filename, self.export_format) + self.log.info( + "Converting notebook %s to %s", notebook_filename, self.export_format + ) else: self.log.info("Converting notebook into %s", self.export_format) @@ -600,7 +618,6 @@ def convert_notebooks(self): "Please specify an output format with '--to '." f"\nThe following formats are available: {get_export_names()}" ) - raise ValueError(msg) # initialize the exporter cls = get_exporter(self.export_format) @@ -648,7 +665,9 @@ def document_config_options(self): preprocessor, postprocessor, and other sections. """ categories = { - category: [c for c in self._classes_inc_parents() if category in c.__name__.lower()] + category: [ + c for c in self._classes_inc_parents() if category in c.__name__.lower() + ] for category in [ "app", "exporter", @@ -658,7 +677,9 @@ def document_config_options(self): ] } accounted_for = {c for category in categories.values() for c in category} - categories["other"] = [c for c in self._classes_inc_parents() if c not in accounted_for] + categories["other"] = [ + c for c in self._classes_inc_parents() if c not in accounted_for + ] header = dedent( """ @@ -672,7 +693,9 @@ def document_config_options(self): sections += header.format(section=category.title()) if category in ["exporter", "preprocessor", "writer"]: sections += f".. image:: _static/{category}_inheritance.png\n\n" - sections += "\n".join(c.class_config_rst_doc() for c in categories[category]) + sections += "\n".join( + c.class_config_rst_doc() for c in categories[category] + ) return sections.replace(" : ", r" \: ") From ffd6ab4760b552dc7c66865072696522b92baae2 Mon Sep 17 00:00:00 2001 From: Joe Scott <17139430+thisiswhereitype@users.noreply.github.com> Date: Wed, 15 May 2024 20:42:20 +0100 Subject: [PATCH 4/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nbconvert/nbconvertapp.py | 37 +++++++++---------------------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/nbconvert/nbconvertapp.py b/nbconvert/nbconvertapp.py index a1186f681..7f5b4ba36 100755 --- a/nbconvert/nbconvertapp.py +++ b/nbconvert/nbconvertapp.py @@ -347,9 +347,7 @@ def _writer_class_changed(self, change): help="""PostProcessor class used to write the results of the conversion""" ).tag(config=True) - postprocessor_aliases = { - "serve": "nbconvert.postprocessors.serve.ServePostProcessor" - } + postprocessor_aliases = {"serve": "nbconvert.postprocessors.serve.ServePostProcessor"} postprocessor_factory = Type(None, allow_none=True) @observe("postprocessor_class") @@ -417,9 +415,7 @@ def init_notebooks(self): # Use glob to find matching filenames. Allow the user to convert # notebooks without having to type the extension. globbed_files = glob.glob(pattern, recursive=self.recursive_glob) - globbed_files.extend( - glob.glob(pattern + ".ipynb", recursive=self.recursive_glob) - ) + globbed_files.extend(glob.glob(pattern + ".ipynb", recursive=self.recursive_glob)) if not globbed_files: self.log.warning("pattern %r matched no files", pattern) @@ -433,10 +429,7 @@ def init_writer(self): self._writer_class_changed({"new": self.writer_class}) if self.writer_factory: self.writer = self.writer_factory(parent=self) - if ( - hasattr(self.writer, "build_directory") - and self.writer.build_directory != "" - ): + if hasattr(self.writer, "build_directory") and self.writer.build_directory != "": self.use_output_suffix = False def init_postprocessor(self): @@ -513,17 +506,13 @@ def export_single_notebook(self, notebook_filename, resources, input_buffer=None """ try: if input_buffer is not None: - output, resources = self.exporter.from_file( - input_buffer, resources=resources - ) + output, resources = self.exporter.from_file(input_buffer, resources=resources) else: output, resources = self.exporter.from_filename( notebook_filename, resources=resources ) except ConversionException: - self.log.error( - "Error while converting '%s'", notebook_filename, exc_info=True - ) # noqa: G201 + self.log.error("Error while converting '%s'", notebook_filename, exc_info=True) # noqa: G201 self.exit(1) return output, resources @@ -592,9 +581,7 @@ def convert_single_notebook(self, notebook_filename, input_buffer=None): argument. """ if input_buffer is None: - self.log.info( - "Converting notebook %s to %s", notebook_filename, self.export_format - ) + self.log.info("Converting notebook %s to %s", notebook_filename, self.export_format) else: self.log.info("Converting notebook into %s", self.export_format) @@ -665,9 +652,7 @@ def document_config_options(self): preprocessor, postprocessor, and other sections. """ categories = { - category: [ - c for c in self._classes_inc_parents() if category in c.__name__.lower() - ] + category: [c for c in self._classes_inc_parents() if category in c.__name__.lower()] for category in [ "app", "exporter", @@ -677,9 +662,7 @@ def document_config_options(self): ] } accounted_for = {c for category in categories.values() for c in category} - categories["other"] = [ - c for c in self._classes_inc_parents() if c not in accounted_for - ] + categories["other"] = [c for c in self._classes_inc_parents() if c not in accounted_for] header = dedent( """ @@ -693,9 +676,7 @@ def document_config_options(self): sections += header.format(section=category.title()) if category in ["exporter", "preprocessor", "writer"]: sections += f".. image:: _static/{category}_inheritance.png\n\n" - sections += "\n".join( - c.class_config_rst_doc() for c in categories[category] - ) + sections += "\n".join(c.class_config_rst_doc() for c in categories[category]) return sections.replace(" : ", r" \: ") From 2a72e8927142436d7e55b60b25d8ae2cfe41b8b8 Mon Sep 17 00:00:00 2001 From: Joe Scott <17139430+thisiswhereitype@users.noreply.github.com> Date: Wed, 15 May 2024 20:42:20 +0100 Subject: [PATCH 5/5] Re-added dropped line --- nbconvert/nbconvertapp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nbconvert/nbconvertapp.py b/nbconvert/nbconvertapp.py index 7f5b4ba36..cee32410f 100755 --- a/nbconvert/nbconvertapp.py +++ b/nbconvert/nbconvertapp.py @@ -605,6 +605,7 @@ def convert_notebooks(self): "Please specify an output format with '--to '." f"\nThe following formats are available: {get_export_names()}" ) + raise ValueError(msg) # initialize the exporter cls = get_exporter(self.export_format)