Explicitly use utf-8 encoding for everything (#61)

* Explicitly use utf-8 encoding for everything

* Remove nbconvert<6 requirement

* Add release note

Author: jayqi

HISTORY.md

@@ -1,8 +1,13 @@
 # History
 
+## 0.3.0 (Unreleased)
+
+- Explicitly set all input and output file encodings to UTF-8, which fixes a bug with HTML exports on Windows with `nbconvert` v6.0.
+  - This is not expected to cause any backwards compatibility issues. However, in the _very_ unlikely instance that your `jupyter_notebook_config.py` file or your `nbautoexport.json` config file is Windows-1252-encoded _and_ contains non-ASCII characters, you will need to convert them to UTF-8. ([#57](https://github.com/drivendataorg/nbautoexport/issues/57), [#61](https://github.com/drivendataorg/nbautoexport/pull/61))
+
 ## 0.2.1 (2020-09-18)
 
- - `nbconvert` released verion 6, which breaks some functionality. Pinning to `nbconvert<6` until we can address [#57](https://github.com/drivendataorg/nbautoexport/issues/57).
+ - `nbconvert` released version 6, which may break HTML exports on Windows. Pinning to `nbconvert<6` until we can address [#57](https://github.com/drivendataorg/nbautoexport/issues/57).
 
 ## 0.2.0 (2020-09-04)
 

nbautoexport/export.py

@@ -37,9 +37,9 @@ def postprocess(self, input: str):
             return
 
         # Rewrite converted file to new path, removing cell numbers
-        with input.open("r") as f:
+        with input.open("r", encoding="utf-8") as f:
             text = f.read()
-        with new_path.open("w") as f:
+        with new_path.open("w", encoding="utf-8") as f:
             f.write(re.sub(r"\n#\sIn\[(([0-9]+)|(\s))\]:\n{2}", "", text))
 
         # For some formats, we also need to move the assets directory, for stuff like images
@@ -80,7 +80,7 @@ def post_save(model: dict, os_path: str, contents_manager: FileContentsManager):
 
     if should_convert:
         config = NbAutoexportConfig.parse_file(
-            path=save_progress_indicator, content_type="application/json"
+            path=save_progress_indicator, content_type="application/json", encoding="utf-8"
         )
         export_notebook(os_path, config=config)
 

nbautoexport/jupyter_config.py

@@ -54,15 +54,15 @@ def install_post_save_hook(config_path: Optional[Path] = None):
     if not config_path.exists():
         logger.debug(f"No existing Jupyter configuration detected at {config_path}. Creating...")
         config_path.parent.mkdir(exist_ok=True, parents=True)
-        with config_path.open("w") as fp:
+        with config_path.open("w", encoding="utf-8") as fp:
             fp.write(post_save_hook_initialize_block)
         logger.info("nbautoexport post-save hook installed.")
         return
 
     # If config exists, check for existing nbautoexport initialize block and install as appropriate
     logger.debug(f"Detected existing Jupyter configuration at {config_path}")
 
-    with config_path.open("r") as fp:
+    with config_path.open("r", encoding="utf-8") as fp:
         config = fp.read()
 
     if block_regex.search(config):
@@ -78,7 +78,7 @@ def install_post_save_hook(config_path: Optional[Path] = None):
 
         if parse_version(existing_version) < parse_version(__version__):
             logger.info(f"Updating nbautoexport post-save hook with version {__version__}...")
-            with config_path.open("w") as fp:
+            with config_path.open("w", encoding="utf-8") as fp:
                 # Open as w replaces existing file. We're replacing entire config.
                 fp.write(block_regex.sub(post_save_hook_initialize_block, config))
         else:

nbautoexport/nbautoexport.py

@@ -99,7 +99,9 @@ def clean(
     sentinel_path = directory / SAVE_PROGRESS_INDICATOR_FILE
     validate_sentinel_path(sentinel_path)
 
-    config = NbAutoexportConfig.parse_file(path=sentinel_path, content_type="application/json")
+    config = NbAutoexportConfig.parse_file(
+        path=sentinel_path, content_type="application/json", encoding="utf-8"
+    )
 
     # Combine exclude patterns from config and command-line
     config.clean.exclude.extend(exclude)
@@ -204,7 +206,9 @@ def export(
     # Configuration: input options override existing sentinel file
     if sentinel_path.exists():
         typer.echo(f"Reading existing configuration file from {sentinel_path} ...")
-        config = NbAutoexportConfig.parse_file(path=sentinel_path, content_type="application/json")
+        config = NbAutoexportConfig.parse_file(
+            path=sentinel_path, content_type="application/json", encoding="utf-8"
+        )
 
         # Overrides
         if len(export_formats) > 0:
@@ -338,7 +342,7 @@ def configure(
         (Path(jupyter_config_dir()) / "jupyter_notebook_config.py").expanduser().resolve()
     )
     if jupyter_config_file.exists():
-        with jupyter_config_file.open("r") as fp:
+        with jupyter_config_file.open("r", encoding="utf-8") as fp:
             jupyter_config_text = fp.read()
         if block_regex.search(jupyter_config_text):
             installed = True

nbautoexport/sentinel.py

@@ -60,5 +60,5 @@ def install_sentinel(directory: Path, config: NbAutoexportConfig, overwrite: boo
     else:
         logger.info(f"Creating configuration file at {sentinel_path}")
         logger.info(f"\n{config.json(indent=2)}")
-        with sentinel_path.open("w") as fp:
+        with sentinel_path.open("w", encoding="utf-8") as fp:
             fp.write(config.json(indent=2))

requirements.txt

@@ -1,5 +1,5 @@
 jupyter-contrib-nbextensions>=0.5.1
-nbconvert>=5.6.1,<6
+nbconvert>=5.6.1
 packaging
 pydantic
 typer>=0.3.0

setup.py

@@ -10,7 +10,7 @@
 
 def load_requirements(path: Path):
     requirements = []
-    with path.open("r") as fp:
+    with path.open("r", encoding="utf-8") as fp:
         for line in fp.readlines():
             if line.startswith("-r"):
                 requirements += load_requirements(line.split(" ")[1].strip())

tests/test_cli_clean.py

@@ -58,7 +58,7 @@ def notebooks_dir(tmp_path, notebook_asset):
 def test_clean(notebooks_dir, need_confirmation, organize_by):
     sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
     config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
-    with sentinel_path.open("w") as fp:
+    with sentinel_path.open("w", encoding="utf-8") as fp:
         fp.write(config.json())
 
     if need_confirmation:
@@ -87,7 +87,7 @@ def test_clean_relative(notebooks_dir, organize_by):
     with working_directory(notebooks_dir):
         sentinel_path = Path(SAVE_PROGRESS_INDICATOR_FILE)
         config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
-        with sentinel_path.open("w") as fp:
+        with sentinel_path.open("w", encoding="utf-8") as fp:
             fp.write(config.json())
 
         result = CliRunner().invoke(app, ["clean", "."], input="y")
@@ -119,7 +119,7 @@ def test_clean_relative_subdirectory(notebooks_dir, organize_by):
 
         sentinel_path = subdir / SAVE_PROGRESS_INDICATOR_FILE
         config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
-        with sentinel_path.open("w") as fp:
+        with sentinel_path.open("w", encoding="utf-8") as fp:
             fp.write(config.json())
 
         result = CliRunner().invoke(app, ["clean", "subdir"], input="y")
@@ -168,7 +168,7 @@ def test_clean_exclude(notebooks_dir):
             organize_by="extension",
             clean=CleanConfig(exclude=["keep.txt", "images/*.jpg"]),
         )
-        with sentinel_path.open("w") as fp:
+        with sentinel_path.open("w", encoding="utf-8") as fp:
             fp.write(config.json())
 
         command = ["clean", "subdir", "-e", "**/*.md"]
@@ -199,7 +199,7 @@ def test_clean_exclude(notebooks_dir):
 
 def test_clean_abort(notebooks_dir):
     sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
-    with sentinel_path.open("w") as fp:
+    with sentinel_path.open("w", encoding="utf-8") as fp:
         fp.write(NbAutoexportConfig(export_formats=EXPECTED_FORMATS).json())
 
     starting_files = set(notebooks_dir.glob("**/*"))
@@ -216,7 +216,7 @@ def test_clean_abort(notebooks_dir):
 
 def test_clean_dry_run(notebooks_dir):
     sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
-    with sentinel_path.open("w") as fp:
+    with sentinel_path.open("w", encoding="utf-8") as fp:
         fp.write(NbAutoexportConfig(export_formats=EXPECTED_FORMATS).json())
 
     starting_files = set(notebooks_dir.glob("**/*"))

tests/test_cli_configure.py

@@ -20,7 +20,9 @@ def test_configure_defaults(tmp_path):
     assert result.exit_code == 0
 
     config = NbAutoexportConfig.parse_file(
-        path=tmp_path / SAVE_PROGRESS_INDICATOR_FILE, content_type="application/json"
+        path=tmp_path / SAVE_PROGRESS_INDICATOR_FILE,
+        content_type="application/json",
+        encoding="utf-8",
     )
 
     expected_config = NbAutoexportConfig()
@@ -45,7 +47,9 @@ def test_configure_specified(tmp_path):
     assert result.exit_code == 0
 
     config = NbAutoexportConfig.parse_file(
-        path=tmp_path / SAVE_PROGRESS_INDICATOR_FILE, content_type="application/json"
+        path=tmp_path / SAVE_PROGRESS_INDICATOR_FILE,
+        content_type="application/json",
+        encoding="utf-8",
     )
 
     expected_config = NbAutoexportConfig(
@@ -91,7 +95,7 @@ def test_force_overwrite(tmp_path):
         app, ["configure", str(tmp_path), "-o", "-f", "script", "-f", "html", "-b", "notebook"]
     )
     assert result.exit_code == 0
-    with (tmp_path / ".nbautoexport").open("r") as fp:
+    with (tmp_path / ".nbautoexport").open("r", encoding="utf-8") as fp:
         config = json.load(fp)
 
     expected_config = NbAutoexportConfig(export_formats=["script", "html"], organize_by="notebook")
@@ -120,7 +124,7 @@ def test_configure_oudated_initialize_warning(tmp_path, monkeypatch):
     monkeypatch.setenv("JUPYTER_CONFIG_DIR", str(tmp_path))
 
     jupyter_config_path = tmp_path / "jupyter_notebook_config.py"
-    with jupyter_config_path.open("w") as fp:
+    with jupyter_config_path.open("w", encoding="utf-8") as fp:
         initialize_block = jupyter_config.version_regex.sub(
             "0", jupyter_config.post_save_hook_initialize_block
         )

tests/test_cli_export.py

@@ -95,7 +95,7 @@ def test_export_with_config_no_cli_opts(notebooks_dir, input_type, organize_by):
 
     sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
     config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
-    with sentinel_path.open("w") as fp:
+    with sentinel_path.open("w", encoding="utf-8") as fp:
         fp.write(config.json())
 
     result = CliRunner().invoke(app, ["export", input_path])
@@ -125,7 +125,7 @@ def test_export_with_config_with_cli_opts(notebooks_dir, input_type, organize_by
 
     sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
     written_config = NbAutoexportConfig()
-    with sentinel_path.open("w") as fp:
+    with sentinel_path.open("w", encoding="utf-8") as fp:
         fp.write(written_config.json())
 
     expected_config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
@@ -156,7 +156,7 @@ def test_export_relative(notebooks_dir, input_type, organize_by):
 
         sentinel_path = Path(SAVE_PROGRESS_INDICATOR_FILE)
         config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
-        with sentinel_path.open("w") as fp:
+        with sentinel_path.open("w", encoding="utf-8") as fp:
             fp.write(config.json())
 
         if input_type == "dir":
@@ -190,7 +190,7 @@ def test_clean_relative_subdirectory(notebooks_dir, input_type, organize_by):
 
         sentinel_path = subdir / SAVE_PROGRESS_INDICATOR_FILE
         config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by)
-        with sentinel_path.open("w") as fp:
+        with sentinel_path.open("w", encoding="utf-8") as fp:
             fp.write(config.json())
 
         expected_notebooks = find_notebooks(subdir)

tests/test_cli_install.py

@@ -13,7 +13,7 @@ def test_install_new_config(tmp_path, monkeypatch):
     assert result.exit_code == 0
     assert config_path.exists()
 
-    with config_path.open("r") as fp:
+    with config_path.open("r", encoding="utf-8") as fp:
         config = fp.read()
     assert config == jupyter_config.post_save_hook_initialize_block
 
@@ -23,15 +23,15 @@ def test_install_existing_config(tmp_path, monkeypatch):
 
     config_path = tmp_path / "jupyter_notebook_config.py"
 
-    with config_path.open("w") as fp:
+    with config_path.open("w", encoding="utf-8") as fp:
         fp.write("print('hello world!')")
     assert config_path.exists()
 
     result = CliRunner().invoke(app, ["install"])
     assert result.exit_code == 0
     assert config_path.exists()
 
-    with config_path.open("r") as fp:
+    with config_path.open("r", encoding="utf-8") as fp:
         config = fp.read()
     assert config == (
         "print('hello world!')" + "\n" + jupyter_config.post_save_hook_initialize_block
@@ -45,23 +45,23 @@ def test_install_new_config_with_path(tmp_path):
     assert result.exit_code == 0
     assert config_path.exists()
 
-    with config_path.open("r") as fp:
+    with config_path.open("r", encoding="utf-8") as fp:
         config = fp.read()
     assert config == jupyter_config.post_save_hook_initialize_block
 
 
 def test_install_existing_config_with_path(tmp_path):
     config_path = tmp_path / "nonstandard_config.py"
 
-    with config_path.open("w") as fp:
+    with config_path.open("w", encoding="utf-8") as fp:
         fp.write("print('hello world!')")
     assert config_path.exists()
 
     result = CliRunner().invoke(app, ["install", "--jupyter-config", str(config_path)])
     assert result.exit_code == 0
     assert config_path.exists()
 
-    with config_path.open("r") as fp:
+    with config_path.open("r", encoding="utf-8") as fp:
         config = fp.read()
     assert config == (
         "print('hello world!')" + "\n" + jupyter_config.post_save_hook_initialize_block

tests/test_export.py

@@ -62,7 +62,7 @@ def test_post_save_no_sentinel(notebooks_dir):
 def test_post_save_organize_by_notebook(notebooks_dir):
     notebook_path = notebooks_dir / "the_notebook.ipynb"
     sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
-    with sentinel_path.open("w") as fp:
+    with sentinel_path.open("w", encoding="utf-8") as fp:
         json.dump(
             NbAutoexportConfig(export_formats=["script", "html"], organize_by="notebook").dict(),
             fp,
@@ -84,7 +84,7 @@ def test_post_save_organize_by_notebook(notebooks_dir):
 def test_post_save_organize_by_extension(notebooks_dir):
     notebook_path = notebooks_dir / "the_notebook.ipynb"
     sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
-    with sentinel_path.open("w") as fp:
+    with sentinel_path.open("w", encoding="utf-8") as fp:
         json.dump(
             NbAutoexportConfig(export_formats=["script", "html"], organize_by="extension").dict(),
             fp,
@@ -109,7 +109,7 @@ def test_post_save_type_file(notebooks_dir):
     """Test that post_save should do nothing if model type is 'file'."""
     notebook_path = notebooks_dir / "the_notebook.ipynb"
     sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
-    with sentinel_path.open("w") as fp:
+    with sentinel_path.open("w", encoding="utf-8") as fp:
         json.dump(NbAutoexportConfig().dict(), fp)
 
     assert notebook_path.exists()
@@ -125,7 +125,7 @@ def test_post_save_type_directory(notebooks_dir):
     """Test that post_save should do nothing if model type is 'directory'."""
     notebook_path = notebooks_dir / "the_notebook.ipynb"
     sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE
-    with sentinel_path.open("w") as fp:
+    with sentinel_path.open("w", encoding="utf-8") as fp:
         json.dump(NbAutoexportConfig().dict(), fp)
 
     assert notebook_path.exists()