Skip to content

Commit

Permalink
[SC-93969] Deprecate deploy in favor of create and edit and update CL…
Browse files Browse the repository at this point in the history
…I messages to reflect backend behavior (#437)

* Deprecate deploy in favor of create and edit.
* update CLI messages to reflect backend behavior.

Testing:
- unittests
- Manually running the commands and verifying

Co-authored-by: Jonathan Seidman <71357890+jaseidman@users.noreply.github.com>
  • Loading branch information
pradeepgv-db and jaseidman committed Apr 1, 2022
1 parent 1e852b5 commit 5444cb8
Show file tree
Hide file tree
Showing 5 changed files with 466 additions and 234 deletions.
8 changes: 7 additions & 1 deletion databricks_cli/click_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,13 @@ class SecretPrincipalClickType(ParamType):

class PipelineSpecClickType(ParamType):
name = 'SPEC'
help = 'The path to the pipelines deployment spec file.'
help = '[Deprecated] Use the settings option instead. \n' + \
'The path to the pipelines settings file.'


class PipelineSettingClickType(ParamType):
name = 'SETTINGS'
help = 'The path to the pipelines settings file.'


class PipelineIdClickType(ParamType):
Expand Down
28 changes: 15 additions & 13 deletions databricks_cli/pipelines/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,14 @@ def __init__(self, api_client):
self.client = DeltaPipelinesService(api_client)
self.dbfs_client = DbfsApi(api_client)

def create(self, spec, spec_dir, allow_duplicate_names, headers=None):
data = self._upload_libraries_and_update_spec(spec, spec_dir)
def create(self, settings, settings_dir, allow_duplicate_names, headers=None):
data = self._upload_libraries_and_update_settings(settings, settings_dir)
data['allow_duplicate_names'] = allow_duplicate_names
return self.client.client.perform_query('POST', '/pipelines', data=data,
headers=headers)

def deploy(self, spec, spec_dir, allow_duplicate_names, headers=None):
data = self._upload_libraries_and_update_spec(spec, spec_dir)
def edit(self, settings, settings_dir, allow_duplicate_names, headers=None):
data = self._upload_libraries_and_update_settings(settings, settings_dir)
data['allow_duplicate_names'] = allow_duplicate_names
pipeline_id = data['id']
self.client.client.perform_query('PUT', '/pipelines/{}'.format(pipeline_id), data=data,
Expand Down Expand Up @@ -86,14 +86,15 @@ def start_update(self, pipeline_id, full_refresh=None, headers=None):
def stop(self, pipeline_id, headers=None):
self.client.stop(pipeline_id, headers)

def _upload_libraries_and_update_spec(self, spec, spec_dir):
spec = copy.deepcopy(spec)
lib_objects = LibraryObject.from_json(spec.get('libraries', []))
def _upload_libraries_and_update_settings(self, settings, settings_dir):
settings = copy.deepcopy(settings)
lib_objects = LibraryObject.from_json(settings.get('libraries', []))
local_lib_objects, external_lib_objects = self._identify_local_libraries(lib_objects)

spec['libraries'] = LibraryObject.to_json(
external_lib_objects + self._upload_local_libraries(spec_dir, local_lib_objects))
return spec
settings['libraries'] = LibraryObject.to_json(
external_lib_objects + self._upload_local_libraries(
settings_dir, local_lib_objects))
return settings

@staticmethod
def _identify_local_libraries(lib_objects):
Expand Down Expand Up @@ -124,9 +125,10 @@ def _identify_local_libraries(lib_objects):
external_lib_objects.append(lib_object)
return local_lib_objects, external_lib_objects

def _upload_local_libraries(self, spec_dir, local_lib_objects):
relative_local_lib_objects = [LibraryObject(llo.lib_type, os.path.join(spec_dir, llo.path))
for llo in local_lib_objects]
def _upload_local_libraries(self, settings_dir, local_lib_objects):
relative_local_lib_objects = [
LibraryObject(
llo.lib_type, os.path.join(settings_dir, llo.path)) for llo in local_lib_objects]
remote_lib_objects = [LibraryObject(rllo.lib_type, self._get_hashed_path(rllo.path))
for rllo in relative_local_lib_objects]
transformed_remote_lib_objects = [LibraryObject(rlo.lib_type, DbfsPath(rlo.path))
Expand Down
Loading

0 comments on commit 5444cb8

Please sign in to comment.