From 4d865db61d474aed65ea18814efb6d84c40579f3 Mon Sep 17 00:00:00 2001 From: Marica Odagaki Date: Mon, 11 Mar 2019 21:13:58 -0700 Subject: [PATCH 1/3] Extract _sync_source_files --- src/elm_doc/project_tasks.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/elm_doc/project_tasks.py b/src/elm_doc/project_tasks.py index c383307..8baa081 100644 --- a/src/elm_doc/project_tasks.py +++ b/src/elm_doc/project_tasks.py @@ -39,8 +39,7 @@ def build_project_docs_json( json.dump(elm_project_with_exposed_modules, f) package_src_dir = build_path / 'src' - for source_dir in project.source_directories: - sync(str(project.path / source_dir), str(package_src_dir), 'sync', create=True) + _sync_source_files(project, package_src_dir) for elm_file_path in package_src_dir.glob('**/*.elm'): if elm_parser.is_port_module(elm_file_path): @@ -65,6 +64,16 @@ def _run_elm_make(elm_path: Path, output_path: Path, build_path: Path): ) +def _sync_source_files(project: ElmProject, target_directory: Path) -> None: + '''Copy source files to a single directory. This meets the requirement of Elm + that a package project can only have a single source directory and gives + us an isolated environment so that Elm can run in parallel with any invocation + of Elm within the actual project. + ''' + for source_dir in project.source_directories: + sync(str(project.path / source_dir), str(target_directory), 'sync', create=True) + + def create_main_project_tasks( project: ElmProject, project_config: ProjectConfig, From 01acbd9e119c48aef8401c26ee702c252022b2df Mon Sep 17 00:00:00 2001 From: Marica Odagaki Date: Mon, 11 Mar 2019 21:55:30 -0700 Subject: [PATCH 2/3] Switch to rsync Using dirsync naively to copy multiple source directories to a single directory will not 'sync' deletions. We could write our own syncing code, but rsync already does it well. I'm not super happy with the external, non-Python dependency, but rsync is available practically everywhere, so accepting the trade-off seems fine. --- Pipfile | 1 - Pipfile.lock | 21 ++++------- README.md | 2 +- setup.py | 1 - src/elm_doc/cli.py | 23 ++++++++++++ src/elm_doc/project_tasks.py | 15 +++++--- tests/test_project_tasks.py | 68 ++++++++++++++++++++++++++++++++++++ 7 files changed, 110 insertions(+), 21 deletions(-) create mode 100644 tests/test_project_tasks.py diff --git a/Pipfile b/Pipfile index 23d91e9..97c55c8 100644 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,6 @@ doit = "*" retrying = "*" typing = "*" attrs = "*" -dirsync = "*" parsy = "*" requests = "*" diff --git a/Pipfile.lock b/Pipfile.lock index b484df3..1b71564 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "bfc117d6663a9fcf4dc2bd2e804f213d68222b9a57390c7145f543dddd0e3699" + "sha256": "566b5f0fe0fa4a953f49d7c9a7b51f5dee3018fb906dd10c100a17fce4fdafee" }, "pipfile-spec": 6, "requires": { @@ -26,10 +26,10 @@ }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "chardet": { "hashes": [ @@ -53,13 +53,6 @@ ], "version": "==0.8.0" }, - "dirsync": { - "hashes": [ - "sha256:1fec130d5f4a4c4d71d0c69a140e7bf970945b2de8a9e0be664fe969d57480e4" - ], - "index": "pypi", - "version": "==2.2.3" - }, "doit": { "hashes": [ "sha256:52b7ad1c0095425bd719c2fc9fb27af94fb0ee478762c50abf385faaf786d010", @@ -161,10 +154,10 @@ }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "cffi": { "hashes": [ diff --git a/README.md b/README.md index 1179713..582a305 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Generate static documentation of your Elm application project. -Requires Python >= 3.5, and macOS or Linux. It may work on Windows but it's untested. +Requires Python >= 3.5, rsync >= 2.6.7, and macOS or Linux. It may work on Windows but it's untested. Supported Elm versions: diff --git a/setup.py b/setup.py index 6f08305..40ebd02 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,6 @@ install_requires=[ 'attrs', 'click', - 'dirsync', 'doit', 'parsy', 'requests', diff --git a/src/elm_doc/cli.py b/src/elm_doc/cli.py index 84786e2..001db19 100644 --- a/src/elm_doc/cli.py +++ b/src/elm_doc/cli.py @@ -2,8 +2,10 @@ import os import os.path import shutil +import subprocess from pathlib import Path import functools +import re import click from doit.doit_cmd import DoitMain @@ -41,6 +43,20 @@ def validate_elm_path(ctx, param, value): return value +REQUIRED_RSYNC_VERSION = (2, 6, 7) + + +def check_rsync_version() -> bool: + output = subprocess.check_output(['rsync', '--version'], universal_newlines=True) + first_line = output.splitlines()[0] + match = re.search(r'version (?P\d)\.(?P\d)\.(?P\d)', first_line) + if not match: + raise click.Abort( + 'could not extract the version of rsync from: {}'.format(first_line)) + version = (int(match['major']), int(match['minor']), int(match['patch'])) + return version >= REQUIRED_RSYNC_VERSION + + def _resolve_path(path: str) -> Path: # not using Path.resolve() for now because we don't expect strict # existence checking. maybe we should. @@ -141,6 +157,13 @@ def main( include_paths): """Generate static documentation for your Elm project""" + if not shutil.which('rsync'): + raise click.UsageError('this program requires rsync') + + if not check_rsync_version(): + raise click.UsageError('this program requires rsync version {} or greater' + .format('.'.join(REQUIRED_RSYNC_VERSION))) + if not validate and output is None: raise click.BadParameter('please specify --output directory') diff --git a/src/elm_doc/project_tasks.py b/src/elm_doc/project_tasks.py index 8baa081..a097f9d 100644 --- a/src/elm_doc/project_tasks.py +++ b/src/elm_doc/project_tasks.py @@ -1,11 +1,11 @@ from typing import List, Optional -from collections import ChainMap +import os.path from pathlib import Path +from collections import ChainMap import json from tempfile import TemporaryDirectory import subprocess -from dirsync import sync from doit.tools import create_folder, config_changed from elm_doc import elm_platform @@ -64,14 +64,21 @@ def _run_elm_make(elm_path: Path, output_path: Path, build_path: Path): ) +@capture_subprocess_error_as_task_failure def _sync_source_files(project: ElmProject, target_directory: Path) -> None: '''Copy source files to a single directory. This meets the requirement of Elm that a package project can only have a single source directory and gives us an isolated environment so that Elm can run in parallel with any invocation of Elm within the actual project. ''' - for source_dir in project.source_directories: - sync(str(project.path / source_dir), str(target_directory), 'sync', create=True) + target_directory.mkdir(parents=True, exist_ok=True) + sources = ['{}/./'.format(os.path.normpath(source_dir)) + for source_dir in project.source_directories] + subprocess.check_output( + ['rsync', '-a', '--delete', '--recursive'] + sources + [str(target_directory)], + cwd=str(project.path), + stderr=subprocess.STDOUT, + ) def create_main_project_tasks( diff --git a/tests/test_project_tasks.py b/tests/test_project_tasks.py new file mode 100644 index 0000000..b1ebf8b --- /dev/null +++ b/tests/test_project_tasks.py @@ -0,0 +1,68 @@ +from pathlib import Path + +from elm_doc import elm_project +from elm_doc import project_tasks + + +def test_sync_source_files_create_new_file( + tmpdir, elm_version, make_elm_project): + project_dir = make_elm_project( + elm_version, + tmpdir, + sources={ + 'src': [ + 'Main.elm', + ], + }, + copy_elm_stuff=False, + ) + project = elm_project.from_path(Path(str(project_dir))) + target_dir = Path(project_dir / 'tmp') + project_tasks._sync_source_files(project, target_dir) + assert (target_dir / 'Main.elm').exists() + + +def test_sync_source_files_update_file( + tmpdir, elm_version, make_elm_project): + project_dir = make_elm_project( + elm_version, + tmpdir, + sources={ + 'src': [ + 'Main.elm', + ], + }, + copy_elm_stuff=False, + ) + project = elm_project.from_path(Path(str(project_dir))) + target_dir = Path(project_dir / 'tmp') + project_tasks._sync_source_files(project, target_dir) + + main_elm = project_dir.join('src', 'Main.elm') + main_elm.write('updated for testing') + + project_tasks._sync_source_files(project, target_dir) + assert (target_dir / 'Main.elm').read_text() == 'updated for testing' + + +def test_sync_source_files_delete_file( + tmpdir, elm_version, make_elm_project): + project_dir = make_elm_project( + elm_version, + tmpdir, + sources={ + 'src': [ + 'Main.elm', + ], + }, + copy_elm_stuff=False, + ) + project = elm_project.from_path(Path(str(project_dir))) + target_dir = Path(project_dir / 'tmp') + project_tasks._sync_source_files(project, target_dir) + + main_elm = project_dir.join('src', 'Main.elm') + main_elm.remove() + + project_tasks._sync_source_files(project, target_dir) + assert not (target_dir / 'Main.elm').exists() From 97f5dae5d4ea05ba8d50cb559d7fa21f64db243d Mon Sep 17 00:00:00 2001 From: Marica Odagaki Date: Mon, 11 Mar 2019 22:59:31 -0700 Subject: [PATCH 3/3] python 3.5 compatibility --- src/elm_doc/cli.py | 4 +++- tests/test_project_tasks.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/elm_doc/cli.py b/src/elm_doc/cli.py index 001db19..0de1c8f 100644 --- a/src/elm_doc/cli.py +++ b/src/elm_doc/cli.py @@ -53,7 +53,9 @@ def check_rsync_version() -> bool: if not match: raise click.Abort( 'could not extract the version of rsync from: {}'.format(first_line)) - version = (int(match['major']), int(match['minor']), int(match['patch'])) + version = (int(match.group('major')), + int(match.group('minor')), + int(match.group('patch'))) return version >= REQUIRED_RSYNC_VERSION diff --git a/tests/test_project_tasks.py b/tests/test_project_tasks.py index b1ebf8b..a01fb73 100644 --- a/tests/test_project_tasks.py +++ b/tests/test_project_tasks.py @@ -17,7 +17,7 @@ def test_sync_source_files_create_new_file( copy_elm_stuff=False, ) project = elm_project.from_path(Path(str(project_dir))) - target_dir = Path(project_dir / 'tmp') + target_dir = Path(str(project_dir / 'tmp')) project_tasks._sync_source_files(project, target_dir) assert (target_dir / 'Main.elm').exists() @@ -35,7 +35,7 @@ def test_sync_source_files_update_file( copy_elm_stuff=False, ) project = elm_project.from_path(Path(str(project_dir))) - target_dir = Path(project_dir / 'tmp') + target_dir = Path(str(project_dir / 'tmp')) project_tasks._sync_source_files(project, target_dir) main_elm = project_dir.join('src', 'Main.elm') @@ -58,7 +58,7 @@ def test_sync_source_files_delete_file( copy_elm_stuff=False, ) project = elm_project.from_path(Path(str(project_dir))) - target_dir = Path(project_dir / 'tmp') + target_dir = Path(str(project_dir / 'tmp')) project_tasks._sync_source_files(project, target_dir) main_elm = project_dir.join('src', 'Main.elm')