From 990d30c6dda0cc2e52aee65d03f21fff64999b50 Mon Sep 17 00:00:00 2001 From: Jeffrey Breen Date: Tue, 14 Aug 2018 21:04:52 -0400 Subject: [PATCH 1/5] added entry for (Apache) license to classifiers to satisfy pyroma --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 5c338cb1..e01bdc8d 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,7 @@ 'Intended Audience :: System Administrators', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.6', + 'License :: OSI Approved :: Apache Software License' ], keywords='databricks cli', url='https://github.com/databricks/databricks-cli' From b35e519bf7e2ce5d7426add97eb24ebd73d1cc78 Mon Sep 17 00:00:00 2001 From: Jeffrey Breen Date: Tue, 14 Aug 2018 21:05:39 -0400 Subject: [PATCH 2/5] bumped version of tox and added click and tabulate packages to dev-requirements.txt --- dev-requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 8ef288ec..dc009910 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1 +1,3 @@ -tox==2.9.1 +tox==3.2.1 +click==6.7 +tabulate==0.8.2 From 2ef919ec9e1a7a7722b5d3389c1ca039250ab2f4 Mon Sep 17 00:00:00 2001 From: Jeffrey Breen Date: Tue, 14 Aug 2018 21:19:17 -0400 Subject: [PATCH 3/5] bumped package versions in tox-requirements.txt to resolve conflicts for pylint-django & pyflakes; still get warning about prospector requiring higher version of pylint, but all tests pass --- tox-requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tox-requirements.txt b/tox-requirements.txt index 5d37cc4c..00569934 100644 --- a/tox-requirements.txt +++ b/tox-requirements.txt @@ -1,10 +1,10 @@ # Test reqs -prospector[with_pyroma]==0.12.7 -pylint==1.8.2 -pep8-naming==0.5.0 -pytest==3.2.1 +prospector[with_pyroma]==1.1.1 +pylint==1.9.3 +pep8-naming==0.7.0 +pytest==3.7.1 mock==2.0.0 -decorator==4.2.1 -rstcheck==3.2 +decorator==4.3.0 +rstcheck==3.3 pytest-cov codecov From 3f056a70a66dadb42e9c09daecc497815c684fdc Mon Sep 17 00:00:00 2001 From: Jeffrey Breen Date: Tue, 14 Aug 2018 21:21:04 -0400 Subject: [PATCH 4/5] added installation (and removal) of packages in tox-requirements.txt so tests will pass; added --no-cache-dir flag to all pip installs --- Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 190ad6ef..d9d1b93e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,11 +4,15 @@ WORKDIR /usr/src/databricks-cli COPY . . -RUN pip install --upgrade pip && \ - pip install -r dev-requirements.txt && \ +RUN pip install --upgrade --no-cache-dir pip && \ + pip install --no-cache-dir \ + -r dev-requirements.txt \ + -r tox-requirements.txt && \ pip list && \ ./lint.sh && \ - pip install . && \ - pytest tests + pip install --no-cache-dir . && \ + pytest tests && \ + pip uninstall -y \ + -r tox-requirements.txt ENTRYPOINT [ "databricks" ] From d202df60324255e0c5737463494b6b533f70bd7e Mon Sep 17 00:00:00 2001 From: Jeffrey Breen Date: Wed, 12 Dec 2018 17:16:08 -0500 Subject: [PATCH 5/5] first try to add ability to select export format for recursive export_dir --- databricks_cli/workspace/api.py | 4 ++-- databricks_cli/workspace/cli.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/databricks_cli/workspace/api.py b/databricks_cli/workspace/api.py index a9889d2b..a4dcbdbb 100644 --- a/databricks_cli/workspace/api.py +++ b/databricks_cli/workspace/api.py @@ -152,7 +152,7 @@ def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidd click.echo(('{} does not have a valid extension of {}. Skip this file and ' + 'continue.').format(cur_src, extensions)) - def export_workspace_dir(self, source_path, target_path, overwrite): + def export_workspace_dir(self, source_path, target_path, format=WorkspaceFormat.SOURCE, overwrite): if os.path.isfile(target_path): click.echo('{} exists as a file. Skipping this subtree {}' .format(target_path, source_path)) @@ -167,7 +167,7 @@ def export_workspace_dir(self, source_path, target_path, overwrite): elif obj.is_notebook: cur_dst = cur_dst + WorkspaceLanguage.to_extension(obj.language) try: - self.export_workspace(cur_src, cur_dst, WorkspaceFormat.SOURCE, overwrite) + self.export_workspace(cur_src, cur_dst, format, overwrite) click.echo('{} -> {}'.format(cur_src, cur_dst)) except LocalFileExistsException: click.echo('{} already exists locally as {}. Skip.'.format(cur_src, cur_dst)) diff --git a/databricks_cli/workspace/cli.py b/databricks_cli/workspace/cli.py index 19997e9a..ceb5925b 100644 --- a/databricks_cli/workspace/cli.py +++ b/databricks_cli/workspace/cli.py @@ -147,22 +147,24 @@ def delete_cli(api_client, workspace_path, recursive): @click.argument('source_path') @click.argument('target_path') @click.option('--overwrite', '-o', is_flag=True, default=False) +@click.option('--format', '-f', default=WorkspaceFormat.SOURCE, type=FormatClickType()) @debug_option @profile_option @eat_exceptions @provide_api_client -def export_dir_cli(api_client, source_path, target_path, overwrite): +def export_dir_cli(api_client, source_path, target_path, format, overwrite): """ Recursively exports a directory from the Databricks workspace. Only directories and notebooks are exported. Notebooks are always exported in the SOURCE format. Notebooks will also have the extension of .scala, .py, .sql, or .r appended depending on the language type. + ADDED format -- jeffreybreen """ workspace_api = WorkspaceApi(api_client) assert workspace_api.get_status(source_path).is_dir, 'The source path must be a directory. {}' \ .format(source_path) - workspace_api.export_workspace_dir(source_path, target_path, overwrite) + workspace_api.export_workspace_dir(source_path, target_path, format, overwrite) @click.command(context_settings=CONTEXT_SETTINGS,