From 90bc5665573de07f4b77161adef361b484135076 Mon Sep 17 00:00:00 2001 From: hughson simon Date: Mon, 16 May 2022 09:38:50 -0700 Subject: [PATCH 01/20] tableau trigger datasource refresh --- .gitignore | 1 + vendor_blueprints/refresh_datasource.py | 87 +++++++++++++++++++++++++ vendor_blueprints/requirements.txt | 2 + 3 files changed, 90 insertions(+) create mode 100644 vendor_blueprints/refresh_datasource.py diff --git a/.gitignore b/.gitignore index b6e4761..27049fa 100644 --- a/.gitignore +++ b/.gitignore @@ -109,6 +109,7 @@ venv/ ENV/ env.bak/ venv.bak/ +.idea/ # Spyder project settings .spyderproject diff --git a/vendor_blueprints/refresh_datasource.py b/vendor_blueprints/refresh_datasource.py new file mode 100644 index 0000000..d45d97b --- /dev/null +++ b/vendor_blueprints/refresh_datasource.py @@ -0,0 +1,87 @@ +import tableauserverclient as TSC +import argparse +import sys + +EXIT_CODE_FINAL_STATUS_SUCCESS = 0 +EXIT_CODE_UNKNOWN_ERROR = 3 +EXIT_CODE_INVALID_CREDENTIALS = 200 +EXIT_CODE_INVALID_RESOURCE = 201 +EXIT_CODE_FINAL_STATUS_ERRORED = 211 +EXIT_CODE_FINAL_STATUS_CANCELLED = 212 + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--username', dest='username', required=True) + parser.add_argument('--password', dest='password', required=True) + parser.add_argument('--site_id', dest='site_id', required=True) + parser.add_argument('--content_url', dest='content_url', required=True) + parser.add_argument('--datasource_id', dest='datasource_id', required=True) + args = parser.parse_args() + return args + +def authenticate_tableu(username, password, site_id, content_url): + """TSC library to sign in and sign out of Tableau Server and Tableau Online. + + :param username:The name of the user. + :param password:The password of the user. + :param site_id: The site_id for required datasources. ex: ffc7f88a-85a7-48d5-ac03-09ef0a677280 + :param content_url: This corresponds to the contentUrl attribute in the Tableau REST API. + :return: connection object + """ + try: + tableau_auth = TSC.TableauAuth(username, password, site_id=site_id) + server = TSC.Server(content_url, use_server_version=True) + server.auth.sign_in(tableau_auth) + except Exception as e: + print(f'Failed to connect to Tableau.') + print(e) + sys.exit(EXIT_CODE_INVALID_CREDENTIALS) + return server + + +def get_datasource(conenction, datasourceid): + """Returns the specified data source item. + + :param conenction: + :param datasourceid: The datasource_item specifies the data source to update. + :return: datasource : get the data source item to update + """ + try: + datasource = conenction.datasources.get_by_id(datasourceid) + except Exception as e: + print(f'Datasource item may not be valid or Datasource must be retrieved from server first.') + print(e) + sys.exit(EXIT_CODE_INVALID_RESOURCE) + return datasource + +def refresh_datasource(conenction, datasourceobj): + """Refreshes the data of the specified extract. + + :param conenction: + :param datasourceobj: The datasource_item specifies the data source to update. + :return: datasource : datasource object + """ + try: + refreshed_datasource = conenction.datasources.refresh(datasourceobj) + print(f'Datasource refresh trigger successful.') + except Exception as e: + print(f'Refresh error or Extract operation for the datasource is not allowed.') + print(e) + sys.exit(EXIT_CODE_INVALID_RESOURCE) + + return EXIT_CODE_FINAL_STATUS_SUCCESS + + +def main(): + args = get_args() + username = args.username + password = args.password + site_id = args.site_id + content_url=args.content_url + datasource_id = args.datasource_id + conenction =authenticate_tableu(username, password, site_id, content_url) + datasourceobj = get_datasource(conenction, datasource_id) + sys.exit(refresh_datasource(conenction, datasourceobj)) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/vendor_blueprints/requirements.txt b/vendor_blueprints/requirements.txt index 67dfcb2..628cf3c 100644 --- a/vendor_blueprints/requirements.txt +++ b/vendor_blueprints/requirements.txt @@ -1,3 +1,5 @@ # List out the required python packages for any of these blueprints to work here. # Example: # pandas==1.3.5 +requests==2.27.1 +tableauserverclient==0.18.0 From ec7a97cc5f93548ecc67dcd6ee48a7914a589003 Mon Sep 17 00:00:00 2001 From: hughson simon Date: Tue, 17 May 2022 13:24:37 -0700 Subject: [PATCH 02/20] rename folder --- {vendor_blueprints => tableau_blueprints}/__init__.py | 0 {vendor_blueprints => tableau_blueprints}/refresh_datasource.py | 0 {vendor_blueprints => tableau_blueprints}/requirements.txt | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {vendor_blueprints => tableau_blueprints}/__init__.py (100%) rename {vendor_blueprints => tableau_blueprints}/refresh_datasource.py (100%) rename {vendor_blueprints => tableau_blueprints}/requirements.txt (100%) diff --git a/vendor_blueprints/__init__.py b/tableau_blueprints/__init__.py similarity index 100% rename from vendor_blueprints/__init__.py rename to tableau_blueprints/__init__.py diff --git a/vendor_blueprints/refresh_datasource.py b/tableau_blueprints/refresh_datasource.py similarity index 100% rename from vendor_blueprints/refresh_datasource.py rename to tableau_blueprints/refresh_datasource.py diff --git a/vendor_blueprints/requirements.txt b/tableau_blueprints/requirements.txt similarity index 100% rename from vendor_blueprints/requirements.txt rename to tableau_blueprints/requirements.txt From 88c3d4e5e535a533a74ce2c6314ec2352695bb62 Mon Sep 17 00:00:00 2001 From: hughson simon Date: Tue, 17 May 2022 19:50:08 -0700 Subject: [PATCH 03/20] download view from tableau server --- .gitignore | 1 + tableau_blueprints/download_view.py | 133 ++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 tableau_blueprints/download_view.py diff --git a/.gitignore b/.gitignore index 27049fa..a485f18 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.DS_Store # Translations *.mo diff --git a/tableau_blueprints/download_view.py b/tableau_blueprints/download_view.py new file mode 100644 index 0000000..938acf8 --- /dev/null +++ b/tableau_blueprints/download_view.py @@ -0,0 +1,133 @@ +import argparse +import sys + +import tableauserverclient as TSC + +EXIT_CODE_FINAL_STATUS_SUCCESS = 0 +EXIT_CODE_UNKNOWN_ERROR = 3 +EXIT_CODE_INVALID_CREDENTIALS = 200 +EXIT_CODE_INVALID_RESOURCE = 201 +EXIT_CODE_FINAL_STATUS_ERRORED = 211 +EXIT_CODE_FINAL_STATUS_CANCELLED = 212 + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--username', dest='username', required=True) + parser.add_argument('--password', dest='password', required=True) + parser.add_argument('--site_id', dest='site_id', required=True) + parser.add_argument('--server-url', dest='server_url', required=True) + parser.add_argument('--view-name', dest='view_name', required=True) + parser.add_argument('--file-type', choices=['image', 'thumbnail', 'pdf', 'csv'], type=str.lower, required=True) + parser.add_argument('--file-name', dest='file_name', required=True) + parser.add_argument('--file-options', dest='file_options', required=False) + args = parser.parse_args() + return args + + +def authenticate_tableu(username, password, site_id, content_url): + """TSC library to sign in and sign out of Tableau Server and Tableau Online. + + :param username:The name of the user. + :param password:The password of the user. + :param site_id: The site_id for required datasources. ex: ffc7f88a-85a7-48d5-ac03-09ef0a677280 + :param content_url: This corresponds to the contentUrl attribute in the Tableau REST API. + :return: connection object + """ + try: + tableau_auth = TSC.TableauAuth(username, password, site_id=site_id) + server = TSC.Server(content_url, use_server_version=True) + server.auth.sign_in(tableau_auth) + except Exception as e: + print(f'Failed to connect to Tableau.') + print(e) + sys.exit(EXIT_CODE_INVALID_CREDENTIALS) + return server, tableau_auth + + +def validate_get_view(server, tableau_auth, view_name): + """Returns the details of a specific view. + + :param server: + :param tableau_auth: + :param view_name: + :return: + """ + try: + view_id = None + with server.auth.sign_in(tableau_auth): + all_views, pagination_item = server.views.get() + for views in all_views: + if views.name == view_name: + # view_obj = server.views.get_by_id(views.id) + view_id = views.id + except Exception as e: + print(f'View item may not be valid.') + print(e) + sys.exit(EXIT_CODE_INVALID_RESOURCE) + return view_id + + +def download_view_item(server, tableau_auth, filename, filetype, view_id): + """to download a view from Tableau Server + + :param server: + :param tableau_auth: + :param filename: name of the view to be downloaded as. + :param filetype: 'image', 'thumbnail', 'pdf', 'csv' + :return: + """ + try: + with server.auth.sign_in(tableau_auth): + views = server.views.get_by_id(view_id) + if filetype == "image": + server.views.populate_image(views) + with open('./' + filename, 'wb') as f: + f.write(views.image) + + if filetype == "pdf": # Populate and save the CSV data in a file + server.views.populate_pdf(views, req_options=None) + with open('./' + filename, 'wb') as f: + f.write(views.pdf) + + if filetype == "csv": + server.views.populate_pdf(views, req_options=None) + with open('./' + filename, 'wb') as f: + # Perform byte join on the CSV data + f.write(b''.join(views.csv)) + + if filetype == "thumbnail": + server.views.populate_pdf(views, req_options=None) + with open('./' + filename, 'wb') as f: + # Perform byte join on the CSV data + f.write(b''.join(views.csv)) + print("View saved to current directory successfully") + except OSError as e: + print(f'Could not write file:.') + print(e) + sys.exit(EXIT_CODE_INVALID_RESOURCE) + + return True + + +def main(): + args = get_args() + username = args.username + password = args.password + site_id = args.site_id + server_url = args.server_url + view_name = args.view_name + filetype = args.file_type + filename = args.file_name + fileoptions = args.file_options # TODO + server, tableau_auth = authenticate_tableu(username, password, site_id, server_url) + view_id = validate_get_view(server, tableau_auth, view_name) + if view_id is not None: + download_view_item(server, tableau_auth, filename, filetype, view_id) + else: + print(f'View item may not be valid.') + sys.exit(EXIT_CODE_INVALID_RESOURCE) + + +if __name__ == '__main__': + main() From 103dd22180af9602309aa25fffb008e1cbcce148 Mon Sep 17 00:00:00 2001 From: hughson simon Date: Tue, 17 May 2022 20:23:49 -0700 Subject: [PATCH 04/20] refresh datasource using name --- .gitignore | 1 + tableau_blueprints/refresh_datasource.py | 62 +++++++++++++++--------- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 27049fa..a485f18 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.DS_Store # Translations *.mo diff --git a/tableau_blueprints/refresh_datasource.py b/tableau_blueprints/refresh_datasource.py index d45d97b..6bf61df 100644 --- a/tableau_blueprints/refresh_datasource.py +++ b/tableau_blueprints/refresh_datasource.py @@ -9,67 +9,79 @@ EXIT_CODE_FINAL_STATUS_ERRORED = 211 EXIT_CODE_FINAL_STATUS_CANCELLED = 212 + def get_args(): parser = argparse.ArgumentParser() parser.add_argument('--username', dest='username', required=True) parser.add_argument('--password', dest='password', required=True) - parser.add_argument('--site_id', dest='site_id', required=True) - parser.add_argument('--content_url', dest='content_url', required=True) - parser.add_argument('--datasource_id', dest='datasource_id', required=True) + parser.add_argument('--site-id', dest='site_id', required=True) + parser.add_argument('--server-url', dest='server_url', required=True) + parser.add_argument('--datasource-name', dest='datasource_name', required=True) args = parser.parse_args() return args -def authenticate_tableu(username, password, site_id, content_url): + +def authenticate_tableau(username, password, site_id, server_url): """TSC library to sign in and sign out of Tableau Server and Tableau Online. :param username:The name of the user. :param password:The password of the user. :param site_id: The site_id for required datasources. ex: ffc7f88a-85a7-48d5-ac03-09ef0a677280 - :param content_url: This corresponds to the contentUrl attribute in the Tableau REST API. + :param server_url: This corresponds to the contentUrl attribute in the Tableau REST API. :return: connection object """ try: tableau_auth = TSC.TableauAuth(username, password, site_id=site_id) - server = TSC.Server(content_url, use_server_version=True) + server = TSC.Server(server_url, use_server_version=True) server.auth.sign_in(tableau_auth) except Exception as e: print(f'Failed to connect to Tableau.') print(e) sys.exit(EXIT_CODE_INVALID_CREDENTIALS) - return server + return server, tableau_auth -def get_datasource(conenction, datasourceid): +def get_datasource(server, tableau_auth, datasource_name): """Returns the specified data source item. - :param conenction: + :param tableau_auth: :param datasourceid: The datasource_item specifies the data source to update. :return: datasource : get the data source item to update """ try: - datasource = conenction.datasources.get_by_id(datasourceid) + datasource_id = None + with server.auth.sign_in(tableau_auth): + all_datasources, pagination_item = server.datasources.get() + for datasources in all_datasources: + # print(datasources.name) + if datasources.name == datasource_name: + # view_obj = server.datasources.get_by_id(datasourceid) + datasource_id = datasources.id + break except Exception as e: print(f'Datasource item may not be valid or Datasource must be retrieved from server first.') print(e) sys.exit(EXIT_CODE_INVALID_RESOURCE) - return datasource + return datasource_id + -def refresh_datasource(conenction, datasourceobj): +def refresh_datasource(server, tableau_auth, datasourceobj): """Refreshes the data of the specified extract. - :param conenction: + :param tableau_auth: :param datasourceobj: The datasource_item specifies the data source to update. :return: datasource : datasource object """ try: - refreshed_datasource = conenction.datasources.refresh(datasourceobj) - print(f'Datasource refresh trigger successful.') + with server.auth.sign_in(tableau_auth): + refreshed_datasource = server.datasources.refresh(datasourceobj) + print(f'Datasource refresh trigger successful.') except Exception as e: print(f'Refresh error or Extract operation for the datasource is not allowed.') print(e) sys.exit(EXIT_CODE_INVALID_RESOURCE) - return EXIT_CODE_FINAL_STATUS_SUCCESS + return refreshed_datasource.id def main(): @@ -77,11 +89,17 @@ def main(): username = args.username password = args.password site_id = args.site_id - content_url=args.content_url - datasource_id = args.datasource_id - conenction =authenticate_tableu(username, password, site_id, content_url) - datasourceobj = get_datasource(conenction, datasource_id) - sys.exit(refresh_datasource(conenction, datasourceobj)) + server_url = args.server_url + datasource_name = args.datasource_name + server, tableau_auth = authenticate_tableau(username, password, site_id, server_url) + datasource_id = get_datasource(server, tableau_auth, datasource_name) + # sys.exit(refresh_datasource(connection, datasourceobj)) + if datasource_id is not None: + refresh_datasource(server, tableau_auth, datasource_id) + else: + print(f'datasource item may not be valid.') + sys.exit(EXIT_CODE_INVALID_RESOURCE) + if __name__ == '__main__': - main() \ No newline at end of file + main() From d826df7dd8f610f631a05ccc02ba72cfedf94f09 Mon Sep 17 00:00:00 2001 From: hughson simon Date: Tue, 17 May 2022 20:26:05 -0700 Subject: [PATCH 05/20] merge with trigger branch --- tableau_blueprints/download_view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tableau_blueprints/download_view.py b/tableau_blueprints/download_view.py index 938acf8..5020240 100644 --- a/tableau_blueprints/download_view.py +++ b/tableau_blueprints/download_view.py @@ -25,7 +25,7 @@ def get_args(): return args -def authenticate_tableu(username, password, site_id, content_url): +def authenticate_tableau(username, password, site_id, content_url): """TSC library to sign in and sign out of Tableau Server and Tableau Online. :param username:The name of the user. @@ -120,7 +120,7 @@ def main(): filetype = args.file_type filename = args.file_name fileoptions = args.file_options # TODO - server, tableau_auth = authenticate_tableu(username, password, site_id, server_url) + server, tableau_auth = authenticate_tableau(username, password, site_id, server_url) view_id = validate_get_view(server, tableau_auth, view_name) if view_id is not None: download_view_item(server, tableau_auth, filename, filetype, view_id) From 96445370466f22f8397a570ad093b6d10d536aaf Mon Sep 17 00:00:00 2001 From: hughson simon Date: Wed, 18 May 2022 22:35:03 -0700 Subject: [PATCH 06/20] verify job status --- tableau_blueprints/job_status.py | 93 ++++++++++++++++++++++++ tableau_blueprints/refresh_datasource.py | 27 +++---- 2 files changed, 107 insertions(+), 13 deletions(-) create mode 100644 tableau_blueprints/job_status.py diff --git a/tableau_blueprints/job_status.py b/tableau_blueprints/job_status.py new file mode 100644 index 0000000..8f38ff2 --- /dev/null +++ b/tableau_blueprints/job_status.py @@ -0,0 +1,93 @@ +import tableauserverclient as TSC +import argparse +import sys + +EXIT_CODE_FINAL_STATUS_SUCCESS = 0 +EXIT_CODE_UNKNOWN_ERROR = 3 +EXIT_CODE_INVALID_CREDENTIALS = 200 +EXIT_CODE_INVALID_RESOURCE = 201 +EXIT_CODE_FINAL_STATUS_ERRORED = 211 +EXIT_CODE_FINAL_STATUS_CANCELLED = 212 + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--username', dest='username', required=True) + parser.add_argument('--password', dest='password', required=True) + parser.add_argument('--site-id', dest='site_id', required=True) + parser.add_argument('--server-url', dest='server_url', required=True) + parser.add_argument('--job-id', dest='job_id', required=True) + args = parser.parse_args() + return args + + +def authenticate_tableau(username, password, site_id, server_url): + """TSC library to sign in and sign out of Tableau Server and Tableau Online. + + :param username:The name of the user. + :param password:The password of the user. + :param site_id: The site_id for required datasources. ex: ffc7f88a-85a7-48d5-ac03-09ef0a677280 + :param server_url: This corresponds to the contentUrl attribute in the Tableau REST API. + :return: connection object + """ + try: + tableau_auth = TSC.TableauAuth(username, password, site_id=site_id) + server = TSC.Server(server_url, use_server_version=True) + connection = server.auth.sign_in(tableau_auth) + except Exception as e: + print(f'Failed to connect to Tableau.') + print(e) + sys.exit(EXIT_CODE_INVALID_CREDENTIALS) + return server, connection + + +def get_job_status(server, connection, job_id): + """Gets information about the specified job. + + :param connection: + :param job_id: he job_id specifies the id of the job that is returned from an asynchronous task. + :return: jobinfo : RefreshExtract, finish_code + """ + try: + datasource_id = None + with connection: + jobinfo = server.jobs.get_by_id(job_id) + except Exception as e: + print(f'Job Resource Not Found.') + print(e) + sys.exit(EXIT_CODE_INVALID_RESOURCE) + return jobinfo + + +def refresh_datasource(server, connection, datasourceobj): + """Refreshes the data of the specified extract. + + :param connection: + :param datasourceobj: The datasource_item specifies the data source to update. + :return: datasource : datasource object + """ + try: + with connection: + refreshed_datasource = server.datasources.refresh(datasourceobj) + print(f'Datasource refresh trigger successful.') + except Exception as e: + print(f'Refresh error or Extract operation for the datasource is not allowed.') + print(e) + sys.exit(EXIT_CODE_INVALID_RESOURCE) + + return refreshed_datasource.id + + +def main(): + args = get_args() + username = args.username + password = args.password + site_id = args.site_id + server_url = args.server_url + job_id = args.job_id + server, connection = authenticate_tableau(username, password, site_id, server_url) + get_job_status(server, connection, job_id) + + +if __name__ == '__main__': + main() diff --git a/tableau_blueprints/refresh_datasource.py b/tableau_blueprints/refresh_datasource.py index 6bf61df..0b804b5 100644 --- a/tableau_blueprints/refresh_datasource.py +++ b/tableau_blueprints/refresh_datasource.py @@ -33,29 +33,27 @@ def authenticate_tableau(username, password, site_id, server_url): try: tableau_auth = TSC.TableauAuth(username, password, site_id=site_id) server = TSC.Server(server_url, use_server_version=True) - server.auth.sign_in(tableau_auth) + connection = server.auth.sign_in(tableau_auth) except Exception as e: print(f'Failed to connect to Tableau.') print(e) sys.exit(EXIT_CODE_INVALID_CREDENTIALS) - return server, tableau_auth + return server, connection -def get_datasource(server, tableau_auth, datasource_name): +def get_datasource_id(server, connection, datasource_name): """Returns the specified data source item. - :param tableau_auth: + :param connection: :param datasourceid: The datasource_item specifies the data source to update. :return: datasource : get the data source item to update """ try: datasource_id = None - with server.auth.sign_in(tableau_auth): + with connection: all_datasources, pagination_item = server.datasources.get() for datasources in all_datasources: - # print(datasources.name) if datasources.name == datasource_name: - # view_obj = server.datasources.get_by_id(datasourceid) datasource_id = datasources.id break except Exception as e: @@ -65,15 +63,15 @@ def get_datasource(server, tableau_auth, datasource_name): return datasource_id -def refresh_datasource(server, tableau_auth, datasourceobj): +def refresh_datasource(server, connection, datasourceobj): """Refreshes the data of the specified extract. - :param tableau_auth: + :param connection: :param datasourceobj: The datasource_item specifies the data source to update. :return: datasource : datasource object """ try: - with server.auth.sign_in(tableau_auth): + with connection: refreshed_datasource = server.datasources.refresh(datasourceobj) print(f'Datasource refresh trigger successful.') except Exception as e: @@ -91,11 +89,14 @@ def main(): site_id = args.site_id server_url = args.server_url datasource_name = args.datasource_name - server, tableau_auth = authenticate_tableau(username, password, site_id, server_url) - datasource_id = get_datasource(server, tableau_auth, datasource_name) + server, connection = authenticate_tableau(username, password, site_id, server_url) + datasource_id = get_datasource_id(server, connection, datasource_name) # sys.exit(refresh_datasource(connection, datasourceobj)) if datasource_id is not None: - refresh_datasource(server, tableau_auth, datasource_id) + # TODO + # calling method twice, somehow the sign in context manager is not able to authenticate. Need to investigate. + server, connection = authenticate_tableau(username, password, site_id, server_url) + refresh_datasource(server, connection, datasource_id) else: print(f'datasource item may not be valid.') sys.exit(EXIT_CODE_INVALID_RESOURCE) From 4250a2f723a3e7780485f501c356cd1563a6ba03 Mon Sep 17 00:00:00 2001 From: hughson simon Date: Thu, 19 May 2022 15:54:46 -0700 Subject: [PATCH 07/20] bug fixes with pdf and csv --- tableau_blueprints/download_view.py | 39 +++++++++++++++-------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/tableau_blueprints/download_view.py b/tableau_blueprints/download_view.py index 5020240..be969d4 100644 --- a/tableau_blueprints/download_view.py +++ b/tableau_blueprints/download_view.py @@ -15,7 +15,7 @@ def get_args(): parser = argparse.ArgumentParser() parser.add_argument('--username', dest='username', required=True) parser.add_argument('--password', dest='password', required=True) - parser.add_argument('--site_id', dest='site_id', required=True) + parser.add_argument('--site-id', dest='site_id', required=True) parser.add_argument('--server-url', dest='server_url', required=True) parser.add_argument('--view-name', dest='view_name', required=True) parser.add_argument('--file-type', choices=['image', 'thumbnail', 'pdf', 'csv'], type=str.lower, required=True) @@ -37,29 +37,28 @@ def authenticate_tableau(username, password, site_id, content_url): try: tableau_auth = TSC.TableauAuth(username, password, site_id=site_id) server = TSC.Server(content_url, use_server_version=True) - server.auth.sign_in(tableau_auth) + connection = server.auth.sign_in(tableau_auth) except Exception as e: print(f'Failed to connect to Tableau.') print(e) sys.exit(EXIT_CODE_INVALID_CREDENTIALS) - return server, tableau_auth + return server, connection -def validate_get_view(server, tableau_auth, view_name): +def validate_get_view(server, connection, view_name): """Returns the details of a specific view. :param server: - :param tableau_auth: + :param connection: :param view_name: :return: """ try: view_id = None - with server.auth.sign_in(tableau_auth): + with connection: all_views, pagination_item = server.views.get() for views in all_views: if views.name == view_name: - # view_obj = server.views.get_by_id(views.id) view_id = views.id except Exception as e: print(f'View item may not be valid.') @@ -68,40 +67,39 @@ def validate_get_view(server, tableau_auth, view_name): return view_id -def download_view_item(server, tableau_auth, filename, filetype, view_id): +def download_view_item(server, connection, filename, filetype, view_id, view_name): """to download a view from Tableau Server :param server: - :param tableau_auth: + :param connection: :param filename: name of the view to be downloaded as. :param filetype: 'image', 'thumbnail', 'pdf', 'csv' :return: """ try: - with server.auth.sign_in(tableau_auth): + with connection: views = server.views.get_by_id(view_id) if filetype == "image": server.views.populate_image(views) with open('./' + filename, 'wb') as f: f.write(views.image) - if filetype == "pdf": # Populate and save the CSV data in a file + if filetype == "pdf": # Populate and save the PDF data in a file server.views.populate_pdf(views, req_options=None) with open('./' + filename, 'wb') as f: f.write(views.pdf) if filetype == "csv": - server.views.populate_pdf(views, req_options=None) + server.views.populate_csv(views, req_options=None) with open('./' + filename, 'wb') as f: # Perform byte join on the CSV data f.write(b''.join(views.csv)) if filetype == "thumbnail": - server.views.populate_pdf(views, req_options=None) + server.views.populate_preview_image(views, req_options=None) with open('./' + filename, 'wb') as f: - # Perform byte join on the CSV data - f.write(b''.join(views.csv)) - print("View saved to current directory successfully") + f.write(views.preview_image) + print("View {view_name} saved to current directory successfully") except OSError as e: print(f'Could not write file:.') print(e) @@ -120,10 +118,13 @@ def main(): filetype = args.file_type filename = args.file_name fileoptions = args.file_options # TODO - server, tableau_auth = authenticate_tableau(username, password, site_id, server_url) - view_id = validate_get_view(server, tableau_auth, view_name) + server, connection = authenticate_tableau(username, password, site_id, server_url) + view_id = validate_get_view(server, connection, view_name) if view_id is not None: - download_view_item(server, tableau_auth, filename, filetype, view_id) + # TODO + # calling method twice, somehow the sign in context manager is not able to authenticate. Need to investigate. + server, connection = authenticate_tableau(username, password, site_id, server_url) + download_view_item(server, connection, filename, filetype, view_id, view_name) else: print(f'View item may not be valid.') sys.exit(EXIT_CODE_INVALID_RESOURCE) From 57e22764252a7cab0e8959f87674ffae03380e37 Mon Sep 17 00:00:00 2001 From: hughson simon Date: Thu, 19 May 2022 18:38:42 -0700 Subject: [PATCH 08/20] pr request changes --- tableau_blueprints/download_view.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tableau_blueprints/download_view.py b/tableau_blueprints/download_view.py index be969d4..ca384ea 100644 --- a/tableau_blueprints/download_view.py +++ b/tableau_blueprints/download_view.py @@ -1,5 +1,6 @@ import argparse import sys +import os import tableauserverclient as TSC @@ -7,9 +8,7 @@ EXIT_CODE_UNKNOWN_ERROR = 3 EXIT_CODE_INVALID_CREDENTIALS = 200 EXIT_CODE_INVALID_RESOURCE = 201 -EXIT_CODE_FINAL_STATUS_ERRORED = 211 -EXIT_CODE_FINAL_STATUS_CANCELLED = 212 - +EXIT_CODE_FILE_WRITE_ERROR = 1021 def get_args(): parser = argparse.ArgumentParser() @@ -76,6 +75,7 @@ def download_view_item(server, connection, filename, filetype, view_id, view_nam :param filetype: 'image', 'thumbnail', 'pdf', 'csv' :return: """ + file_path = os.path.dirname(os.path.realpath(__file__)) try: with connection: views = server.views.get_by_id(view_id) @@ -99,11 +99,11 @@ def download_view_item(server, connection, filename, filetype, view_id, view_nam server.views.populate_preview_image(views, req_options=None) with open('./' + filename, 'wb') as f: f.write(views.preview_image) - print("View {view_name} saved to current directory successfully") + print(f'View {view_name} successfully saved to {file_path}') except OSError as e: print(f'Could not write file:.') print(e) - sys.exit(EXIT_CODE_INVALID_RESOURCE) + sys.exit(EXIT_CODE_FILE_WRITE_ERROR) return True From 08fc559db64b16520d3ad303ec841aac904cfb19 Mon Sep 17 00:00:00 2001 From: hughson simon Date: Thu, 19 May 2022 21:07:28 -0700 Subject: [PATCH 09/20] execute job initial commit --- tableau_blueprints/execute_job.py | 39 ++++++++++++++++++++++++ tableau_blueprints/refresh_datasource.py | 21 ++++++++++--- 2 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 tableau_blueprints/execute_job.py diff --git a/tableau_blueprints/execute_job.py b/tableau_blueprints/execute_job.py new file mode 100644 index 0000000..5702120 --- /dev/null +++ b/tableau_blueprints/execute_job.py @@ -0,0 +1,39 @@ +import argparse +import os +import json +import time +import platform +import pickle +import sys + +# Handle import difference between local and github install +try: + import job_status + import refresh_datasource +except BaseException: + from . import job_status + from . import refresh_datasource + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--username', dest='username', required=True) + parser.add_argument('--password', dest='password', required=True) + parser.add_argument('--site-id', dest='site_id', required=True) + parser.add_argument('--server-url', dest='server_url', required=True) + parser.add_argument('--view-name', dest='view_name', required=True) + parser.add_argument('--file-type', choices=['image', 'thumbnail', 'pdf', 'csv'], type=str.lower, required=True) + parser.add_argument('--file-name', dest='file_name', required=True) + parser.add_argument('--file-options', dest='file_options', required=False) + args = parser.parse_args() + return args + + +def main(): + args = get_args() + + artifact_directory_default = f'{os.environ.get("USER")}-artifacts' + base_folder_name = '/Users/shughson/Documents/shipyardapp/blueprints/tableau-blueprints/' + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/tableau_blueprints/refresh_datasource.py b/tableau_blueprints/refresh_datasource.py index 0b804b5..737c440 100644 --- a/tableau_blueprints/refresh_datasource.py +++ b/tableau_blueprints/refresh_datasource.py @@ -1,6 +1,7 @@ import tableauserverclient as TSC import argparse import sys +import json EXIT_CODE_FINAL_STATUS_SUCCESS = 0 EXIT_CODE_UNKNOWN_ERROR = 3 @@ -21,6 +22,16 @@ def get_args(): return args +def write_json_to_file(json_object, file_name): + with open(file_name, 'w') as f: + f.write( + json.dumps( + json_object, + ensure_ascii=False, + indent=4)) + print(f'Response stored at {file_name}') + + def authenticate_tableau(username, password, site_id, server_url): """TSC library to sign in and sign out of Tableau Server and Tableau Online. @@ -63,7 +74,7 @@ def get_datasource_id(server, connection, datasource_name): return datasource_id -def refresh_datasource(server, connection, datasourceobj): +def refresh_datasource(server, connection, datasourceobj,datasource_name): """Refreshes the data of the specified extract. :param connection: @@ -73,7 +84,7 @@ def refresh_datasource(server, connection, datasourceobj): try: with connection: refreshed_datasource = server.datasources.refresh(datasourceobj) - print(f'Datasource refresh trigger successful.') + print(f'Datasource {datasource_name} was successfully triggered.') except Exception as e: print(f'Refresh error or Extract operation for the datasource is not allowed.') print(e) @@ -91,14 +102,14 @@ def main(): datasource_name = args.datasource_name server, connection = authenticate_tableau(username, password, site_id, server_url) datasource_id = get_datasource_id(server, connection, datasource_name) - # sys.exit(refresh_datasource(connection, datasourceobj)) if datasource_id is not None: # TODO # calling method twice, somehow the sign in context manager is not able to authenticate. Need to investigate. server, connection = authenticate_tableau(username, password, site_id, server_url) - refresh_datasource(server, connection, datasource_id) + refresh_datasource(server, connection, datasource_id, datasource_name) else: - print(f'datasource item may not be valid.') + print(f'{datasource_name} could not be found or your user does not have access. ' + f'Please check for typos and ensure that the name you provide matches exactly (case senstive)') sys.exit(EXIT_CODE_INVALID_RESOURCE) From f066129b25515d401f35e4e4c437004a6d1c9ea3 Mon Sep 17 00:00:00 2001 From: hughson simon Date: Sun, 22 May 2022 18:13:58 -0700 Subject: [PATCH 10/20] view download using workbook, refresh datasource using project name, pick job id from pickle --- tableau_blueprints/download_view.py | 63 +++++++++---- tableau_blueprints/execute_job.py | 39 -------- tableau_blueprints/job_status.py | 89 +++++++++++++----- tableau_blueprints/refresh_datasource.py | 115 ++++++++++++++++++----- 4 files changed, 198 insertions(+), 108 deletions(-) delete mode 100644 tableau_blueprints/execute_job.py diff --git a/tableau_blueprints/download_view.py b/tableau_blueprints/download_view.py index ca384ea..8b49045 100644 --- a/tableau_blueprints/download_view.py +++ b/tableau_blueprints/download_view.py @@ -20,6 +20,7 @@ def get_args(): parser.add_argument('--file-type', choices=['image', 'thumbnail', 'pdf', 'csv'], type=str.lower, required=True) parser.add_argument('--file-name', dest='file_name', required=True) parser.add_argument('--file-options', dest='file_options', required=False) + parser.add_argument('--workbook-name', dest='workbook_name', required=True) args = parser.parse_args() return args @@ -36,6 +37,7 @@ def authenticate_tableau(username, password, site_id, content_url): try: tableau_auth = TSC.TableauAuth(username, password, site_id=site_id) server = TSC.Server(content_url, use_server_version=True) + server.version = '3.15' connection = server.auth.sign_in(tableau_auth) except Exception as e: print(f'Failed to connect to Tableau.') @@ -44,25 +46,49 @@ def authenticate_tableau(username, password, site_id, content_url): return server, connection -def validate_get_view(server, connection, view_name): - """Returns the details of a specific view. +def validate_get_view(server, connection, view_name, workbook_name): + """Validate workbook and returns the details of a specific view. :param server: - :param connection: - :param view_name: - :return: + :param connection: Tableau connection object + :param view_name: The name of the view. + :param workbook_name: The name of the workbook associated with the view. + :return: view_id: details of a specific view. """ try: view_id = None + req_option = TSC.RequestOptions() + req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, + TSC.RequestOptions.Operator.Equals, + workbook_name)) with connection: - all_views, pagination_item = server.views.get() - for views in all_views: - if views.name == view_name: - view_id = views.id + all_workbooks = server.workbooks.get(req_options=req_option) + workbook_id= [workbook.id for workbook in all_workbooks[0] if workbook.name == workbook_name] + if workbook_id[0] is not None: + # get the workbook item + workbook = server.workbooks.get_by_id(workbook_id[0]) + + # get the view information + server.workbooks.populate_views(workbook) + + # print information about the views for the work item + view_info=[view.id for view in workbook.views if view.name == view_name] + else: + print(f'{view_name} could not be found for the given workbook {workbook_name}.' + f'Please check for typos and ensure that the name you provide matches exactly (case senstive)') + sys.exit(EXIT_CODE_INVALID_RESOURCE) + except Exception as e: - print(f'View item may not be valid.') + print(f'{view_name} could not be found for the given workbook {workbook_name}.' + f'Please check for typos and ensure that the name you provide matches exactly (case senstive)') print(e) sys.exit(EXIT_CODE_INVALID_RESOURCE) + if len(view_info)<=0: + print(f'{view_name} could not be found or your user does not have access. ' + f'Please check for typos and ensure that the name you provide matches exactly (case senstive)') + sys.exit(EXIT_CODE_INVALID_RESOURCE) + else: + view_id=view_info[0] return view_id @@ -70,7 +96,7 @@ def download_view_item(server, connection, filename, filetype, view_id, view_nam """to download a view from Tableau Server :param server: - :param connection: + :param connection: Tableau connection object :param filename: name of the view to be downloaded as. :param filetype: 'image', 'thumbnail', 'pdf', 'csv' :return: @@ -117,17 +143,14 @@ def main(): view_name = args.view_name filetype = args.file_type filename = args.file_name + workbook_name = args.workbook_name fileoptions = args.file_options # TODO server, connection = authenticate_tableau(username, password, site_id, server_url) - view_id = validate_get_view(server, connection, view_name) - if view_id is not None: - # TODO - # calling method twice, somehow the sign in context manager is not able to authenticate. Need to investigate. - server, connection = authenticate_tableau(username, password, site_id, server_url) - download_view_item(server, connection, filename, filetype, view_id, view_name) - else: - print(f'View item may not be valid.') - sys.exit(EXIT_CODE_INVALID_RESOURCE) + view_id = validate_get_view(server, connection, view_name, workbook_name) + # TODO + # calling method twice, somehow the sign in context manager is not able to authenticate. Need to investigate. + server, connection = authenticate_tableau(username, password, site_id, server_url) + download_view_item(server, connection, filename, filetype, view_id, view_name) if __name__ == '__main__': diff --git a/tableau_blueprints/execute_job.py b/tableau_blueprints/execute_job.py deleted file mode 100644 index 5702120..0000000 --- a/tableau_blueprints/execute_job.py +++ /dev/null @@ -1,39 +0,0 @@ -import argparse -import os -import json -import time -import platform -import pickle -import sys - -# Handle import difference between local and github install -try: - import job_status - import refresh_datasource -except BaseException: - from . import job_status - from . import refresh_datasource - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--username', dest='username', required=True) - parser.add_argument('--password', dest='password', required=True) - parser.add_argument('--site-id', dest='site_id', required=True) - parser.add_argument('--server-url', dest='server_url', required=True) - parser.add_argument('--view-name', dest='view_name', required=True) - parser.add_argument('--file-type', choices=['image', 'thumbnail', 'pdf', 'csv'], type=str.lower, required=True) - parser.add_argument('--file-name', dest='file_name', required=True) - parser.add_argument('--file-options', dest='file_options', required=False) - args = parser.parse_args() - return args - - -def main(): - args = get_args() - - artifact_directory_default = f'{os.environ.get("USER")}-artifacts' - base_folder_name = '/Users/shughson/Documents/shipyardapp/blueprints/tableau-blueprints/' - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/tableau_blueprints/job_status.py b/tableau_blueprints/job_status.py index 8f38ff2..9e91760 100644 --- a/tableau_blueprints/job_status.py +++ b/tableau_blueprints/job_status.py @@ -1,6 +1,10 @@ +from httprequest_blueprints import execute_request + import tableauserverclient as TSC import argparse import sys +import os +import pickle EXIT_CODE_FINAL_STATUS_SUCCESS = 0 EXIT_CODE_UNKNOWN_ERROR = 3 @@ -8,6 +12,10 @@ EXIT_CODE_INVALID_RESOURCE = 201 EXIT_CODE_FINAL_STATUS_ERRORED = 211 EXIT_CODE_FINAL_STATUS_CANCELLED = 212 +EXIT_CODE_STATUS_INCOMPLETE = 210 +EXIT_CODE_JOB_NOT_FOUND = 404 +EXIT_CODE_JOB_CACNELLED = 403 +EXIT_CODE_GENERIC_QUERY_JOB_ERROR = 400 def get_args(): @@ -41,52 +49,83 @@ def authenticate_tableau(username, password, site_id, server_url): return server, connection +def determine_run_status(run_details_response): + """Job status response handler. + The finishCode indicates the status of the job: 0 for success, 1 for error, or 2 for cancelled. + + :param run_details_response: + :return: + """ + run_id = run_details_response.id + if run_details_response.finish_code == 0: + if run_details_response.progress == "Pending": + print(f'Tableau reports that the run {run_id} in Pending status.') + exit_code = EXIT_CODE_STATUS_INCOMPLETE + elif run_details_response.progress == "Cancelled": + print(f'Tableau reports that run {run_id} was cancelled.') + exit_code = EXIT_CODE_JOB_CACNELLED + elif run_details_response.progress == "Failed": + print(f'Tableau reports that run {run_id} was Failed.') + exit_code = EXIT_CODE_FINAL_STATUS_ERRORED + elif run_details_response.progress == "InProgress": + print(f'Tableau reports that run {run_id} is in InProgress.') + exit_code = EXIT_CODE_STATUS_INCOMPLETE + else: + print(f'Tableau reports that run {run_id} was successful.') + exit_code = EXIT_CODE_FINAL_STATUS_SUCCESS + elif run_details_response.finish_code == 1: + print(f'Tableau reports that the job {run_id} is exited with error.') + exit_code = EXIT_CODE_GENERIC_QUERY_JOB_ERROR + else: + print(f'Tableau reports that the job {run_id} was cancelled.') + exit_code = EXIT_CODE_STATUS_INCOMPLETE + return exit_code + + def get_job_status(server, connection, job_id): """Gets information about the specified job. :param connection: - :param job_id: he job_id specifies the id of the job that is returned from an asynchronous task. + :param job_id: the job_id specifies the id of the job that is returned from an asynchronous task. :return: jobinfo : RefreshExtract, finish_code """ try: - datasource_id = None with connection: jobinfo = server.jobs.get_by_id(job_id) except Exception as e: - print(f'Job Resource Not Found.') + print(f'Job {job_id} Resource Not Found.') print(e) - sys.exit(EXIT_CODE_INVALID_RESOURCE) + sys.exit(EXIT_CODE_JOB_NOT_FOUND) return jobinfo -def refresh_datasource(server, connection, datasourceobj): - """Refreshes the data of the specified extract. - - :param connection: - :param datasourceobj: The datasource_item specifies the data source to update. - :return: datasource : datasource object - """ - try: - with connection: - refreshed_datasource = server.datasources.refresh(datasourceobj) - print(f'Datasource refresh trigger successful.') - except Exception as e: - print(f'Refresh error or Extract operation for the datasource is not allowed.') - print(e) - sys.exit(EXIT_CODE_INVALID_RESOURCE) - - return refreshed_datasource.id - - def main(): args = get_args() username = args.username password = args.password site_id = args.site_id server_url = args.server_url - job_id = args.job_id + + artifact_directory_default = f'{os.environ.get("USER")}-artifacts' + + base_folder_name = execute_request.clean_folder_name( + f'{os.environ.get("SHIPYARD_ARTIFACTS_DIRECTORY", artifact_directory_default)}/tableau-blueprints/') + + pickle_folder_name = execute_request.clean_folder_name( + f'{base_folder_name}/variables') + + execute_request.create_folder_if_dne(pickle_folder_name) + pickle_file_name = execute_request.combine_folder_and_file_name( + pickle_folder_name, 'job_id.pickle') + + if args.job_id: + job_id = args.job_id + else: + with open(pickle_file_name, 'rb') as f: + job_id = pickle.load(f) + server, connection = authenticate_tableau(username, password, site_id, server_url) - get_job_status(server, connection, job_id) + sys.exit(get_job_status(server, connection, job_id)) if __name__ == '__main__': diff --git a/tableau_blueprints/refresh_datasource.py b/tableau_blueprints/refresh_datasource.py index 737c440..77fe30f 100644 --- a/tableau_blueprints/refresh_datasource.py +++ b/tableau_blueprints/refresh_datasource.py @@ -1,14 +1,26 @@ +from httprequest_blueprints import execute_request + import tableauserverclient as TSC import argparse import sys import json +import time +import os +import pickle + +# Handle import difference between local and github install +try: + import job_status +except BaseException: + from . import job_status EXIT_CODE_FINAL_STATUS_SUCCESS = 0 EXIT_CODE_UNKNOWN_ERROR = 3 EXIT_CODE_INVALID_CREDENTIALS = 200 EXIT_CODE_INVALID_RESOURCE = 201 -EXIT_CODE_FINAL_STATUS_ERRORED = 211 -EXIT_CODE_FINAL_STATUS_CANCELLED = 212 +EXIT_CODE_JOB_NOT_FOUND = 404 +EXIT_CODE_JOB_CACNELLED = 403 +EXIT_CODE_GENERIC_QUERY_JOB_ERROR = 400 def get_args(): @@ -18,6 +30,12 @@ def get_args(): parser.add_argument('--site-id', dest='site_id', required=True) parser.add_argument('--server-url', dest='server_url', required=True) parser.add_argument('--datasource-name', dest='datasource_name', required=True) + parser.add_argument('--project-name', dest='project_name', required=True) + parser.add_argument('--check-status', dest='check_status', default='TRUE', + choices={ + 'TRUE', + 'FALSE'}, + required=False) args = parser.parse_args() return args @@ -44,6 +62,7 @@ def authenticate_tableau(username, password, site_id, server_url): try: tableau_auth = TSC.TableauAuth(username, password, site_id=site_id) server = TSC.Server(server_url, use_server_version=True) + server.version = '3.15' connection = server.auth.sign_in(tableau_auth) except Exception as e: print(f'Failed to connect to Tableau.') @@ -52,35 +71,54 @@ def authenticate_tableau(username, password, site_id, server_url): return server, connection -def get_datasource_id(server, connection, datasource_name): - """Returns the specified data source item. +def get_datasource_id(server, connection, datasource_name, project_name): + """Returns the specified data source item - :param connection: - :param datasourceid: The datasource_item specifies the data source to update. - :return: datasource : get the data source item to update + :param server: + :param connection: Tableau connection object + :param datasource_name: The name of the data source. + :param project_name: The name of the project associated with the data source. + :return: datasource : get the data source item to refresh """ try: datasource_id = None with connection: - all_datasources, pagination_item = server.datasources.get() + req_option = TSC.RequestOptions() + req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, + TSC.RequestOptions.Operator.Equals, + datasource_name)) + all_datasources, pagination_item = server.datasources.get(req_options=req_option) for datasources in all_datasources: if datasources.name == datasource_name: - datasource_id = datasources.id - break + if datasources.project_name == project_name: + datasource_id = datasources.id + break + else: + print(f'{datasource_name} could not be found for the give project {project_name}.') + sys.exit(EXIT_CODE_INVALID_RESOURCE) except Exception as e: print(f'Datasource item may not be valid or Datasource must be retrieved from server first.') print(e) sys.exit(EXIT_CODE_INVALID_RESOURCE) + + if datasource_id is None: + print(f'{datasource_name} could not be found or your user does not have access. ' + f'Please check for typos and ensure that the name you provide matches exactly (case senstive)') + sys.exit(EXIT_CODE_INVALID_RESOURCE) + return datasource_id -def refresh_datasource(server, connection, datasourceobj,datasource_name): +def refresh_datasource(server, connection, datasourceobj, datasource_name): """Refreshes the data of the specified extract. - :param connection: + :param server: + :param connection: Tableau connection object :param datasourceobj: The datasource_item specifies the data source to update. - :return: datasource : datasource object + :param datasource_name: The name of the data source. + :return: refreshed id : refreshed object """ + try: with connection: refreshed_datasource = server.datasources.refresh(datasourceobj) @@ -90,7 +128,7 @@ def refresh_datasource(server, connection, datasourceobj,datasource_name): print(e) sys.exit(EXIT_CODE_INVALID_RESOURCE) - return refreshed_datasource.id + return refreshed_datasource def main(): @@ -100,17 +138,46 @@ def main(): site_id = args.site_id server_url = args.server_url datasource_name = args.datasource_name + project_name = args.project_name + check_status = execute_request.convert_to_boolean(args.check_status) server, connection = authenticate_tableau(username, password, site_id, server_url) - datasource_id = get_datasource_id(server, connection, datasource_name) - if datasource_id is not None: - # TODO - # calling method twice, somehow the sign in context manager is not able to authenticate. Need to investigate. - server, connection = authenticate_tableau(username, password, site_id, server_url) - refresh_datasource(server, connection, datasource_id, datasource_name) - else: - print(f'{datasource_name} could not be found or your user does not have access. ' - f'Please check for typos and ensure that the name you provide matches exactly (case senstive)') - sys.exit(EXIT_CODE_INVALID_RESOURCE) + datasource_id = get_datasource_id(server, connection, datasource_name, project_name) + + # TODO + # calling method twice, somehow the sign in context manager is not able to authenticate. Need to investigate. + server, connection = authenticate_tableau(username, password, site_id, server_url) + refreshed_datasource = refresh_datasource(server, connection, datasource_id, datasource_name) + job_id = refreshed_datasource.id + + artifact_directory_default = f'{os.environ.get("USER")}-artifacts' + base_folder_name = execute_request.clean_folder_name( + f'{os.environ.get("SHIPYARD_ARTIFACTS_DIRECTORY", artifact_directory_default)}/tableau-blueprints/') + + folder_name = f'{base_folder_name}/responses', + file_name = f'job_{job_id}_response.json' + + combined_name = execute_request.combine_folder_and_file_name(folder_name, file_name) + # save the refresh response . check for json + write_json_to_file(refreshed_datasource, combined_name) + + pickle_folder_name = execute_request.clean_folder_name( + f'{base_folder_name}/variables') + execute_request.create_folder_if_dne(pickle_folder_name) + pickle_file_name = execute_request.combine_folder_and_file_name( + pickle_folder_name, 'job_id.pickle') + with open(pickle_file_name, 'wb') as f: + pickle.dump(job_id, f) + + if check_status: + try: + # `wait_for_job` will throw if the job isn't executed successfully + server.jobs.wait_for_job(refreshed_datasource) + time.sleep(30) + server, connection = authenticate_tableau(username, password, site_id, server_url) + sys.exit(job_status.get_job_status(server, connection, job_id)) + except: + print(f'Tableau reports that the job {job_id} is exited as incomplete.') + sys.exit(EXIT_CODE_GENERIC_QUERY_JOB_ERROR) if __name__ == '__main__': From 5de49e1bf84f07dd470c1944f7f8426bde7a90a8 Mon Sep 17 00:00:00 2001 From: Blake Burch Date: Wed, 8 Jun 2022 21:50:33 -0500 Subject: [PATCH 11/20] Updated authorization logic --- tableau_blueprints/authorization.py | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tableau_blueprints/authorization.py diff --git a/tableau_blueprints/authorization.py b/tableau_blueprints/authorization.py new file mode 100644 index 0000000..160b1f1 --- /dev/null +++ b/tableau_blueprints/authorization.py @@ -0,0 +1,49 @@ +import sys +import tableauserverclient as TSC + +EXIT_CODE_INVALID_CREDENTIALS = 200 + + +def connect_to_tableau( + username, + password, + site_id, + server_url, + sign_in_method): + """TSC library to sign in and sign out of Tableau Server and Tableau Online. + + :param username:The username or access token name of the user. + :param password:The password or access token of the user. + :param site_id: The site_id for required datasources. ex: ffc7f88a-85a7-48d5-ac03-09ef0a677280 + :param server_url: This corresponds to the contentUrl attribute in the Tableau REST API. + :param sign_in_method: Whether to log in with username_password or access_token. + :return: connection object + """ + if sign_in_method == 'username_password': + tableau_auth = TSC.TableauAuth(username, password, site_id=site_id) + + if sign_in_method == 'access_token': + tableau_auth = TSC.PersonalAccessTokenAuth( + token_name=username, + personal_access_token=password, + site_id=site_id) + + try: + # Make sure we use an updated version of the rest apis. + server = TSC.Server( + server_url, + use_server_version=True, + ) + connection = server.auth.sign_in(tableau_auth) + print("Successfully authenticated with Tableau.") + except Exception as e: + print(f'Failed to connect to Tableau.') + if sign_in_method == 'username_password': + print('Invalid username or password. Please check for typos and try again.') + if sign_in_method == 'access_token': + print( + 'Invalid token name or access token. Please check for typos and try again.') + print(e) + sys.exit(EXIT_CODE_INVALID_CREDENTIALS) + + return server, connection From eb8ff19c656477260dde154de00401175dc834f8 Mon Sep 17 00:00:00 2001 From: Blake Burch Date: Wed, 8 Jun 2022 21:50:41 -0500 Subject: [PATCH 12/20] Updated download logic --- tableau_blueprints/download_view.py | 278 +++++++++++++++++----------- 1 file changed, 170 insertions(+), 108 deletions(-) diff --git a/tableau_blueprints/download_view.py b/tableau_blueprints/download_view.py index 8b49045..3786c75 100644 --- a/tableau_blueprints/download_view.py +++ b/tableau_blueprints/download_view.py @@ -1,138 +1,170 @@ import argparse import sys import os +import code +import shipyard_utils as shipyard import tableauserverclient as TSC +try: + import authorization +except BaseException: + from . import authorization + EXIT_CODE_FINAL_STATUS_SUCCESS = 0 EXIT_CODE_UNKNOWN_ERROR = 3 EXIT_CODE_INVALID_CREDENTIALS = 200 -EXIT_CODE_INVALID_RESOURCE = 201 -EXIT_CODE_FILE_WRITE_ERROR = 1021 +EXIT_CODE_INVALID_PROJECT = 201 +EXIT_CODE_INVALID_WORKBOOK = 202 +EXIT_CODE_INVALID_VIEW = 203 +EXIT_CODE_FILE_WRITE_ERROR = 204 + def get_args(): parser = argparse.ArgumentParser() parser.add_argument('--username', dest='username', required=True) parser.add_argument('--password', dest='password', required=True) + parser.add_argument( + '--sign-in-method', + dest='sign_in_method', + default='username_password', + choices={ + 'username_password', + 'access_token'}, + required=False) parser.add_argument('--site-id', dest='site_id', required=True) parser.add_argument('--server-url', dest='server_url', required=True) parser.add_argument('--view-name', dest='view_name', required=True) - parser.add_argument('--file-type', choices=['image', 'thumbnail', 'pdf', 'csv'], type=str.lower, required=True) - parser.add_argument('--file-name', dest='file_name', required=True) + parser.add_argument( + '--file-type', + dest='file_type', + choices=[ + 'png', + 'pdf', + 'csv'], + type=str.lower, + required=True) + parser.add_argument( + '--destination-file-name', + dest='destination_file_name', + default='output.csv', + required=True) + parser.add_argument( + '--destination-folder-name', + dest='destination_folder_name', + default='', + required=False) parser.add_argument('--file-options', dest='file_options', required=False) parser.add_argument('--workbook-name', dest='workbook_name', required=True) + parser.add_argument('--project-name', dest='project_name', required=True) args = parser.parse_args() return args -def authenticate_tableau(username, password, site_id, content_url): - """TSC library to sign in and sign out of Tableau Server and Tableau Online. - - :param username:The name of the user. - :param password:The password of the user. - :param site_id: The site_id for required datasources. ex: ffc7f88a-85a7-48d5-ac03-09ef0a677280 - :param content_url: This corresponds to the contentUrl attribute in the Tableau REST API. - :return: connection object +def get_project_id(server, project_name): """ - try: - tableau_auth = TSC.TableauAuth(username, password, site_id=site_id) - server = TSC.Server(content_url, use_server_version=True) - server.version = '3.15' - connection = server.auth.sign_in(tableau_auth) - except Exception as e: - print(f'Failed to connect to Tableau.') - print(e) - sys.exit(EXIT_CODE_INVALID_CREDENTIALS) - return server, connection - + Looks up and returns the project_id of the project_name that was specified. + """ + req_option = TSC.RequestOptions() + req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, + TSC.RequestOptions.Operator.Equals, + project_name)) + + project_matches = server.projects.get(req_options=req_option) + if len(project_matches[0]) == 1: + project_id = project_matches[0][0].id + else: + print( + f'{project_name} could not be found. Please check for typos and ensure that the name you provide matches exactly (case sensitive)') + sys.exit(EXIT_CODE_INVALID_PROJECT) + return project_id -def validate_get_view(server, connection, view_name, workbook_name): - """Validate workbook and returns the details of a specific view. - :param server: - :param connection: Tableau connection object - :param view_name: The name of the view. - :param workbook_name: The name of the workbook associated with the view. - :return: view_id: details of a specific view. +def get_workbook_id(server, project_id, workbook_name): """ - try: - view_id = None - req_option = TSC.RequestOptions() - req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, - TSC.RequestOptions.Operator.Equals, - workbook_name)) - with connection: - all_workbooks = server.workbooks.get(req_options=req_option) - workbook_id= [workbook.id for workbook in all_workbooks[0] if workbook.name == workbook_name] - if workbook_id[0] is not None: - # get the workbook item - workbook = server.workbooks.get_by_id(workbook_id[0]) - - # get the view information - server.workbooks.populate_views(workbook) - - # print information about the views for the work item - view_info=[view.id for view in workbook.views if view.name == view_name] - else: - print(f'{view_name} could not be found for the given workbook {workbook_name}.' - f'Please check for typos and ensure that the name you provide matches exactly (case senstive)') - sys.exit(EXIT_CODE_INVALID_RESOURCE) - - except Exception as e: - print(f'{view_name} could not be found for the given workbook {workbook_name}.' - f'Please check for typos and ensure that the name you provide matches exactly (case senstive)') - print(e) - sys.exit(EXIT_CODE_INVALID_RESOURCE) - if len(view_info)<=0: - print(f'{view_name} could not be found or your user does not have access. ' - f'Please check for typos and ensure that the name you provide matches exactly (case senstive)') - sys.exit(EXIT_CODE_INVALID_RESOURCE) - else: - view_id=view_info[0] + Looks up and returns the workbook_id of the workbook_name that was specified, filtered by project_id matches. + """ + req_option = TSC.RequestOptions() + req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, + TSC.RequestOptions.Operator.Equals, + workbook_name)) + + workbook_matches = server.workbooks.get(req_options=req_option) + # We can't filter by project_id in the initial request, + # so we have to find all name matches and look for a project_id match. + workbook_id = None + for workbook in workbook_matches[0]: + if workbook.project_id == project_id: + workbook_id = workbook.id + if workbook_id is None: + print( + f'{workbook_name} could not be found that lives in the project you specified. Please check for typos and ensure that the name(s) you provide match exactly (case sensitive)') + sys.exit(EXIT_CODE_INVALID_WORKBOOK) + return workbook_id + + +def get_view_id(server, project_id, workbook_id, view_name): + """ + Looks up and returns the view_id of the view_name that was specified, filtered by project_id AND workbook_id matches. + """ + req_option = TSC.RequestOptions() + req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, + TSC.RequestOptions.Operator.Equals, + view_name)) + + view_matches = server.views.get(req_options=req_option) + # We can't filter by project_id or workbook_id in the initial request, + # so we have to find all name matches and look for those matches. + view_id = None + for view in view_matches[0]: + if view.project_id == project_id: + if view.workbook_id == workbook_id: + view_id = view.id + if view_id is None: + print( + f'{view_name} could not be found that lives in the project and workbook you specified. Please check for typos and ensure that the name(s) you provide match exactly (case sensitive)') + sys.exit(EXIT_CODE_INVALID_VIEW) return view_id -def download_view_item(server, connection, filename, filetype, view_id, view_name): - """to download a view from Tableau Server - - :param server: - :param connection: Tableau connection object - :param filename: name of the view to be downloaded as. - :param filetype: 'image', 'thumbnail', 'pdf', 'csv' - :return: +def generate_view_content(server, view_id, file_type): + """ + Given a specific view_id, populate the view and return the bytes necessary for creating the file. + """ + view_object = server.views.get_by_id(view_id) + if file_type == 'png': + server.views.populate_image(view_object) + view_content = view_object.image + if file_type == 'pdf': + server.views.populate_pdf(view_object, req_options=None) + view_content = view_object.pdf + if file_type == 'csv': + server.views.populate_csv(view_object, req_options=None) + view_content = view_object.csv + return view_content + + +def write_view_content_to_file( + destination_full_path, + view_content, + file_type, + view_name): + """ + Write the byte contents to the specified file path. """ - file_path = os.path.dirname(os.path.realpath(__file__)) try: - with connection: - views = server.views.get_by_id(view_id) - if filetype == "image": - server.views.populate_image(views) - with open('./' + filename, 'wb') as f: - f.write(views.image) - - if filetype == "pdf": # Populate and save the PDF data in a file - server.views.populate_pdf(views, req_options=None) - with open('./' + filename, 'wb') as f: - f.write(views.pdf) - - if filetype == "csv": - server.views.populate_csv(views, req_options=None) - with open('./' + filename, 'wb') as f: - # Perform byte join on the CSV data - f.write(b''.join(views.csv)) - - if filetype == "thumbnail": - server.views.populate_preview_image(views, req_options=None) - with open('./' + filename, 'wb') as f: - f.write(views.preview_image) - print(f'View {view_name} successfully saved to {file_path}') + with open(destination_full_path, 'wb') as f: + if file_type == 'csv': + f.writelines(view_content) + else: + f.write(view_content) + print( + f'Successfully downloaded {view_name} to {destination_full_path}') except OSError as e: - print(f'Could not write file:.') + print(f'Could not write file: {destination_full_path}') print(e) sys.exit(EXIT_CODE_FILE_WRITE_ERROR) - return True - def main(): args = get_args() @@ -140,17 +172,47 @@ def main(): password = args.password site_id = args.site_id server_url = args.server_url + sign_in_method = args.sign_in_method view_name = args.view_name - filetype = args.file_type - filename = args.file_name + file_type = args.file_type + project_name = args.project_name workbook_name = args.workbook_name - fileoptions = args.file_options # TODO - server, connection = authenticate_tableau(username, password, site_id, server_url) - view_id = validate_get_view(server, connection, view_name, workbook_name) - # TODO - # calling method twice, somehow the sign in context manager is not able to authenticate. Need to investigate. - server, connection = authenticate_tableau(username, password, site_id, server_url) - download_view_item(server, connection, filename, filetype, view_id, view_name) + + # Set all file parameters + destination_file_name = args.destination_file_name + destination_folder_name = shipyard.files.clean_folder_name( + args.destination_folder_name) + destination_full_path = shipyard.files.combine_folder_and_file_name( + folder_name=destination_folder_name, file_name=destination_file_name) + + server, connection = authorization.connect_to_tableau( + username, + password, + site_id, + server_url, + sign_in_method) + + with connection: + project_id = get_project_id(server=server, project_name=project_name) + workbook_id = get_workbook_id( + server=server, + project_id=project_id, + workbook_name=workbook_name) + view_id = get_view_id( + server=server, + project_id=project_id, + workbook_id=workbook_id, + view_name=view_name) + + view_content = generate_view_content( + server=server, view_id=view_id, file_type=file_type) + shipyard.files.create_folder_if_dne( + destination_folder_name=destination_folder_name) + write_view_content_to_file( + destination_full_path=destination_full_path, + view_content=view_content, + file_type=file_type, + view_name=view_name) if __name__ == '__main__': From 8541cebe137ab6c5f5e633614b230ff2eb790f83 Mon Sep 17 00:00:00 2001 From: Blake Burch Date: Thu, 9 Jun 2022 21:11:13 -0500 Subject: [PATCH 13/20] Updates to triggering and checking job --- tableau_blueprints/job_status.py | 147 +++++++-------- tableau_blueprints/refresh_datasource.py | 222 ++++++++++------------- 2 files changed, 162 insertions(+), 207 deletions(-) diff --git a/tableau_blueprints/job_status.py b/tableau_blueprints/job_status.py index 9e91760..f6616e5 100644 --- a/tableau_blueprints/job_status.py +++ b/tableau_blueprints/job_status.py @@ -1,21 +1,17 @@ -from httprequest_blueprints import execute_request - import tableauserverclient as TSC import argparse import sys import os import pickle +import shipyard_utils as shipyard +import code -EXIT_CODE_FINAL_STATUS_SUCCESS = 0 -EXIT_CODE_UNKNOWN_ERROR = 3 -EXIT_CODE_INVALID_CREDENTIALS = 200 -EXIT_CODE_INVALID_RESOURCE = 201 -EXIT_CODE_FINAL_STATUS_ERRORED = 211 -EXIT_CODE_FINAL_STATUS_CANCELLED = 212 -EXIT_CODE_STATUS_INCOMPLETE = 210 -EXIT_CODE_JOB_NOT_FOUND = 404 -EXIT_CODE_JOB_CACNELLED = 403 -EXIT_CODE_GENERIC_QUERY_JOB_ERROR = 400 +try: + import errors + import authorization +except BaseException: + from . import errors + from . import authorization def get_args(): @@ -24,108 +20,89 @@ def get_args(): parser.add_argument('--password', dest='password', required=True) parser.add_argument('--site-id', dest='site_id', required=True) parser.add_argument('--server-url', dest='server_url', required=True) - parser.add_argument('--job-id', dest='job_id', required=True) + parser.add_argument( + '--sign-in-method', + dest='sign_in_method', + default='username_password', + choices={ + 'username_password', + 'access_token'}, + required=False) + parser.add_argument('--job-id', dest='job_id', required=False) args = parser.parse_args() return args -def authenticate_tableau(username, password, site_id, server_url): - """TSC library to sign in and sign out of Tableau Server and Tableau Online. - - :param username:The name of the user. - :param password:The password of the user. - :param site_id: The site_id for required datasources. ex: ffc7f88a-85a7-48d5-ac03-09ef0a677280 - :param server_url: This corresponds to the contentUrl attribute in the Tableau REST API. - :return: connection object +def get_job_info(server, job_id): + """ + Gets information about the specified job_id. """ try: - tableau_auth = TSC.TableauAuth(username, password, site_id=site_id) - server = TSC.Server(server_url, use_server_version=True) - connection = server.auth.sign_in(tableau_auth) + job_info = server.jobs.get_by_id(job_id) except Exception as e: - print(f'Failed to connect to Tableau.') + print(f'Job {job_id} was not found.') print(e) - sys.exit(EXIT_CODE_INVALID_CREDENTIALS) - return server, connection + sys.exit(errors.EXIT_CODE_JOB_NOT_FOUND) + return job_info -def determine_run_status(run_details_response): - """Job status response handler. - The finishCode indicates the status of the job: 0 for success, 1 for error, or 2 for cancelled. +def determine_job_status(server, job_id): + """ + Job status response handler. - :param run_details_response: - :return: + The finishCode indicates the status of the job: -1 for incomplete, 0 for success, 1 for error, or 2 for cancelled. """ - run_id = run_details_response.id - if run_details_response.finish_code == 0: - if run_details_response.progress == "Pending": - print(f'Tableau reports that the run {run_id} in Pending status.') - exit_code = EXIT_CODE_STATUS_INCOMPLETE - elif run_details_response.progress == "Cancelled": - print(f'Tableau reports that run {run_id} was cancelled.') - exit_code = EXIT_CODE_JOB_CACNELLED - elif run_details_response.progress == "Failed": - print(f'Tableau reports that run {run_id} was Failed.') - exit_code = EXIT_CODE_FINAL_STATUS_ERRORED - elif run_details_response.progress == "InProgress": - print(f'Tableau reports that run {run_id} is in InProgress.') - exit_code = EXIT_CODE_STATUS_INCOMPLETE + job_info = get_job_info(server, job_id) + if job_info.finish_code == -1: + if job_info.started_at is None: + print( + f'Tableau reports that the job {job_id} has not yet started.') else: - print(f'Tableau reports that run {run_id} was successful.') - exit_code = EXIT_CODE_FINAL_STATUS_SUCCESS - elif run_details_response.finish_code == 1: - print(f'Tableau reports that the job {run_id} is exited with error.') - exit_code = EXIT_CODE_GENERIC_QUERY_JOB_ERROR + print( + f'Tableau reports that the job {job_id} is not yet complete.') + exit_code = errors.EXIT_CODE_STATUS_INCOMPLETE + elif job_info.finish_code == 0: + print(f'Tableau reports that job {job_id} was successful.') + exit_code = errors.EXIT_CODE_FINAL_STATUS_SUCCESS + elif job_info.finish_code == 1: + print(f'Tableau reports that job {job_id} errored.') + exit_code = errors.EXIT_CODE_FINAL_STATUS_ERRORED + elif job_info.finish_code == 2: + print(f'Tableau reports that job {job_id} was cancelled.') + exit_code = errors.EXIT_CODE_FINAL_STATUS_CANCELLED else: - print(f'Tableau reports that the job {run_id} was cancelled.') - exit_code = EXIT_CODE_STATUS_INCOMPLETE + print(f'Something went wrong when fetching status for job {job_id}') + exit_code = errors.EXIT_CODE_GENERIC_QUERY_JOB_ERROR return exit_code -def get_job_status(server, connection, job_id): - """Gets information about the specified job. - - :param connection: - :param job_id: the job_id specifies the id of the job that is returned from an asynchronous task. - :return: jobinfo : RefreshExtract, finish_code - """ - try: - with connection: - jobinfo = server.jobs.get_by_id(job_id) - except Exception as e: - print(f'Job {job_id} Resource Not Found.') - print(e) - sys.exit(EXIT_CODE_JOB_NOT_FOUND) - return jobinfo - - def main(): args = get_args() username = args.username password = args.password site_id = args.site_id server_url = args.server_url + sign_in_method = args.sign_in_method - artifact_directory_default = f'{os.environ.get("USER")}-artifacts' - - base_folder_name = execute_request.clean_folder_name( - f'{os.environ.get("SHIPYARD_ARTIFACTS_DIRECTORY", artifact_directory_default)}/tableau-blueprints/') - - pickle_folder_name = execute_request.clean_folder_name( - f'{base_folder_name}/variables') - - execute_request.create_folder_if_dne(pickle_folder_name) - pickle_file_name = execute_request.combine_folder_and_file_name( - pickle_folder_name, 'job_id.pickle') + base_folder_name = shipyard.logs.determine_base_artifact_folder( + 'dbtcloud') + artifact_subfolder_paths = shipyard.logs.determine_artifact_subfolders( + base_folder_name) + shipyard.logs.create_artifacts_folders(artifact_subfolder_paths) if args.job_id: job_id = args.job_id else: - with open(pickle_file_name, 'rb') as f: - job_id = pickle.load(f) + job_id = shipyard.logs.read_pickle_file( + artifact_subfolder_paths, 'job_id') + + server, connection = authorization.connect_to_tableau( + username, password, site_id, server_url, sign_in_method) - server, connection = authenticate_tableau(username, password, site_id, server_url) - sys.exit(get_job_status(server, connection, job_id)) + with connection: + # job_info = get_job_info(server, job_id) + job_status = determine_job_status(server, job_id) + sys.exit(job_status) if __name__ == '__main__': diff --git a/tableau_blueprints/refresh_datasource.py b/tableau_blueprints/refresh_datasource.py index 77fe30f..f805e6b 100644 --- a/tableau_blueprints/refresh_datasource.py +++ b/tableau_blueprints/refresh_datasource.py @@ -1,26 +1,19 @@ -from httprequest_blueprints import execute_request - import tableauserverclient as TSC import argparse import sys -import json import time -import os -import pickle +import shipyard_utils as shipyard +import code # Handle import difference between local and github install try: import job_status + import errors + import authorization except BaseException: from . import job_status - -EXIT_CODE_FINAL_STATUS_SUCCESS = 0 -EXIT_CODE_UNKNOWN_ERROR = 3 -EXIT_CODE_INVALID_CREDENTIALS = 200 -EXIT_CODE_INVALID_RESOURCE = 201 -EXIT_CODE_JOB_NOT_FOUND = 404 -EXIT_CODE_JOB_CACNELLED = 403 -EXIT_CODE_GENERIC_QUERY_JOB_ERROR = 400 + from . import errors + from . import authorization def get_args(): @@ -29,7 +22,18 @@ def get_args(): parser.add_argument('--password', dest='password', required=True) parser.add_argument('--site-id', dest='site_id', required=True) parser.add_argument('--server-url', dest='server_url', required=True) - parser.add_argument('--datasource-name', dest='datasource_name', required=True) + parser.add_argument( + '--sign-in-method', + dest='sign_in_method', + default='username_password', + choices={ + 'username_password', + 'access_token'}, + required=False) + parser.add_argument( + '--datasource-name', + dest='datasource_name', + required=True) parser.add_argument('--project-name', dest='project_name', required=True) parser.add_argument('--check-status', dest='check_status', default='TRUE', choices={ @@ -40,76 +44,50 @@ def get_args(): return args -def write_json_to_file(json_object, file_name): - with open(file_name, 'w') as f: - f.write( - json.dumps( - json_object, - ensure_ascii=False, - indent=4)) - print(f'Response stored at {file_name}') - - -def authenticate_tableau(username, password, site_id, server_url): - """TSC library to sign in and sign out of Tableau Server and Tableau Online. - - :param username:The name of the user. - :param password:The password of the user. - :param site_id: The site_id for required datasources. ex: ffc7f88a-85a7-48d5-ac03-09ef0a677280 - :param server_url: This corresponds to the contentUrl attribute in the Tableau REST API. - :return: connection object +def get_project_id(server, project_name): """ - try: - tableau_auth = TSC.TableauAuth(username, password, site_id=site_id) - server = TSC.Server(server_url, use_server_version=True) - server.version = '3.15' - connection = server.auth.sign_in(tableau_auth) - except Exception as e: - print(f'Failed to connect to Tableau.') - print(e) - sys.exit(EXIT_CODE_INVALID_CREDENTIALS) - return server, connection - - -def get_datasource_id(server, connection, datasource_name, project_name): - """Returns the specified data source item - - :param server: - :param connection: Tableau connection object - :param datasource_name: The name of the data source. - :param project_name: The name of the project associated with the data source. - :return: datasource : get the data source item to refresh + Looks up and returns the project_id of the project_name that was specified. """ - try: - datasource_id = None - with connection: - req_option = TSC.RequestOptions() - req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, - TSC.RequestOptions.Operator.Equals, - datasource_name)) - all_datasources, pagination_item = server.datasources.get(req_options=req_option) - for datasources in all_datasources: - if datasources.name == datasource_name: - if datasources.project_name == project_name: - datasource_id = datasources.id - break - else: - print(f'{datasource_name} could not be found for the give project {project_name}.') - sys.exit(EXIT_CODE_INVALID_RESOURCE) - except Exception as e: - print(f'Datasource item may not be valid or Datasource must be retrieved from server first.') - print(e) - sys.exit(EXIT_CODE_INVALID_RESOURCE) - + req_option = TSC.RequestOptions() + req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, + TSC.RequestOptions.Operator.Equals, + project_name)) + + project_matches = server.projects.get(req_options=req_option) + if len(project_matches[0]) == 1: + project_id = project_matches[0][0].id + else: + print( + f'{project_name} could not be found. Please check for typos and ensure that the name you provide matches exactly (case sensitive)') + sys.exit(errors.EXIT_CODE_INVALID_PROJECT) + return project_id + + +def get_datasource_id(server, project_id, datasource_name): + """ + Looks up and returns the datasource_id of the datasource_name that was specified, filtered by project_id matches. + """ + req_option = TSC.RequestOptions() + req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, + TSC.RequestOptions.Operator.Equals, + datasource_name)) + + datasource_matches = server.datasources.get(req_options=req_option) + + # We can't filter by project_id or project_name in the initial request, + # so we have to find all name matches and look for a project_id match. + datasource_id = None + for datasource in datasource_matches[0]: + if datasource.project_id == project_id: + datasource_id = datasource.id if datasource_id is None: - print(f'{datasource_name} could not be found or your user does not have access. ' - f'Please check for typos and ensure that the name you provide matches exactly (case senstive)') - sys.exit(EXIT_CODE_INVALID_RESOURCE) - + print( + f'{datasource_name} could not be found that lives in the project you specified. Please check for typos and ensure that the name(s) you provide match exactly (case sensitive)') + sys.exit(errors.EXIT_CODE_INVALID_DATASOURCE) return datasource_id -def refresh_datasource(server, connection, datasourceobj, datasource_name): +def refresh_datasource(server, datasource_id, datasource_name): """Refreshes the data of the specified extract. :param server: @@ -120,13 +98,19 @@ def refresh_datasource(server, connection, datasourceobj, datasource_name): """ try: - with connection: - refreshed_datasource = server.datasources.refresh(datasourceobj) - print(f'Datasource {datasource_name} was successfully triggered.') + datasource = server.datasources.get_by_id(datasource_id) + refreshed_datasource = server.datasources.refresh(datasource) + print(f'Datasource {datasource_name} was successfully triggered.') except Exception as e: - print(f'Refresh error or Extract operation for the datasource is not allowed.') + if 'Resource Conflict' in e.args[0]: + print( + f'A refresh or extract operation for the datasource is already underway.') + if 'is not allowed.' in e.args[0]: + print(f'Refresh or extract operation for the datasource is not allowed.') + else: + print(f'An unknown refresh or extract error occurred.') print(e) - sys.exit(EXIT_CODE_INVALID_RESOURCE) + sys.exit(errors.EXIT_CODE_REFRESH_ERROR) return refreshed_datasource @@ -139,45 +123,39 @@ def main(): server_url = args.server_url datasource_name = args.datasource_name project_name = args.project_name - check_status = execute_request.convert_to_boolean(args.check_status) - server, connection = authenticate_tableau(username, password, site_id, server_url) - datasource_id = get_datasource_id(server, connection, datasource_name, project_name) - - # TODO - # calling method twice, somehow the sign in context manager is not able to authenticate. Need to investigate. - server, connection = authenticate_tableau(username, password, site_id, server_url) - refreshed_datasource = refresh_datasource(server, connection, datasource_id, datasource_name) - job_id = refreshed_datasource.id - - artifact_directory_default = f'{os.environ.get("USER")}-artifacts' - base_folder_name = execute_request.clean_folder_name( - f'{os.environ.get("SHIPYARD_ARTIFACTS_DIRECTORY", artifact_directory_default)}/tableau-blueprints/') - - folder_name = f'{base_folder_name}/responses', - file_name = f'job_{job_id}_response.json' - - combined_name = execute_request.combine_folder_and_file_name(folder_name, file_name) - # save the refresh response . check for json - write_json_to_file(refreshed_datasource, combined_name) - - pickle_folder_name = execute_request.clean_folder_name( - f'{base_folder_name}/variables') - execute_request.create_folder_if_dne(pickle_folder_name) - pickle_file_name = execute_request.combine_folder_and_file_name( - pickle_folder_name, 'job_id.pickle') - with open(pickle_file_name, 'wb') as f: - pickle.dump(job_id, f) - - if check_status: - try: - # `wait_for_job` will throw if the job isn't executed successfully - server.jobs.wait_for_job(refreshed_datasource) - time.sleep(30) - server, connection = authenticate_tableau(username, password, site_id, server_url) - sys.exit(job_status.get_job_status(server, connection, job_id)) - except: - print(f'Tableau reports that the job {job_id} is exited as incomplete.') - sys.exit(EXIT_CODE_GENERIC_QUERY_JOB_ERROR) + sign_in_method = args.sign_in_method + should_check_status = shipyard.args.convert_to_boolean(args.check_status) + + base_folder_name = shipyard.logs.determine_base_artifact_folder( + 'dbtcloud') + artifact_subfolder_paths = shipyard.logs.determine_artifact_subfolders( + base_folder_name) + shipyard.logs.create_artifacts_folders(artifact_subfolder_paths) + + server, connection = authorization.connect_to_tableau( + username, password, site_id, server_url, sign_in_method) + + with connection: + project_id = get_project_id(server, project_name) + datasource_id = get_datasource_id( + server, project_id, datasource_name) + refreshed_datasource = refresh_datasource( + server, datasource_id, datasource_name) + job_id = refreshed_datasource.id + + if should_check_status: + try: + # `wait_for_job` will automatically check every few seconds + # and throw if the job isn't executed successfully + print('Waiting for the job to complete...') + server.jobs.wait_for_job(job_id) + exit_code = job_status.determine_job_status(server, job_id) + except BaseException: + exit_code = job_status.determine_job_status(server, job_id) + sys.exit(exit_code) + else: + shipyard.logs.create_pickle_file( + artifact_subfolder_paths, 'job_id', job_id) if __name__ == '__main__': From 8d4aac5b9eac9bfeb6732054e94ece9f8930bb09 Mon Sep 17 00:00:00 2001 From: Blake Burch Date: Thu, 9 Jun 2022 21:13:43 -0500 Subject: [PATCH 14/20] Updated error codes --- tableau_blueprints/download_view.py | 18 ++++++------------ tableau_blueprints/errors.py | 16 ++++++++++++++++ tableau_blueprints/job_status.py | 4 ++-- 3 files changed, 24 insertions(+), 14 deletions(-) create mode 100644 tableau_blueprints/errors.py diff --git a/tableau_blueprints/download_view.py b/tableau_blueprints/download_view.py index 3786c75..395ee99 100644 --- a/tableau_blueprints/download_view.py +++ b/tableau_blueprints/download_view.py @@ -8,16 +8,10 @@ try: import authorization + import errors except BaseException: from . import authorization - -EXIT_CODE_FINAL_STATUS_SUCCESS = 0 -EXIT_CODE_UNKNOWN_ERROR = 3 -EXIT_CODE_INVALID_CREDENTIALS = 200 -EXIT_CODE_INVALID_PROJECT = 201 -EXIT_CODE_INVALID_WORKBOOK = 202 -EXIT_CODE_INVALID_VIEW = 203 -EXIT_CODE_FILE_WRITE_ERROR = 204 + from . import errors def get_args(): @@ -76,7 +70,7 @@ def get_project_id(server, project_name): else: print( f'{project_name} could not be found. Please check for typos and ensure that the name you provide matches exactly (case sensitive)') - sys.exit(EXIT_CODE_INVALID_PROJECT) + sys.exit(errors.EXIT_CODE_INVALID_PROJECT) return project_id @@ -99,7 +93,7 @@ def get_workbook_id(server, project_id, workbook_name): if workbook_id is None: print( f'{workbook_name} could not be found that lives in the project you specified. Please check for typos and ensure that the name(s) you provide match exactly (case sensitive)') - sys.exit(EXIT_CODE_INVALID_WORKBOOK) + sys.exit(errors.EXIT_CODE_INVALID_WORKBOOK) return workbook_id @@ -123,7 +117,7 @@ def get_view_id(server, project_id, workbook_id, view_name): if view_id is None: print( f'{view_name} could not be found that lives in the project and workbook you specified. Please check for typos and ensure that the name(s) you provide match exactly (case sensitive)') - sys.exit(EXIT_CODE_INVALID_VIEW) + sys.exit(errors.EXIT_CODE_INVALID_VIEW) return view_id @@ -163,7 +157,7 @@ def write_view_content_to_file( except OSError as e: print(f'Could not write file: {destination_full_path}') print(e) - sys.exit(EXIT_CODE_FILE_WRITE_ERROR) + sys.exit(errors.EXIT_CODE_FILE_WRITE_ERROR) def main(): diff --git a/tableau_blueprints/errors.py b/tableau_blueprints/errors.py new file mode 100644 index 0000000..b206bf4 --- /dev/null +++ b/tableau_blueprints/errors.py @@ -0,0 +1,16 @@ +EXIT_CODE_FINAL_STATUS_SUCCESS = 0 +EXIT_CODE_UNKNOWN_ERROR = 3 + +EXIT_CODE_FILE_WRITE_ERROR = 100 + +EXIT_CODE_INVALID_CREDENTIALS = 200 +EXIT_CODE_INVALID_PROJECT = 201 +EXIT_CODE_INVALID_WORKBOOK = 202 +EXIT_CODE_INVALID_VIEW = 203 +EXIT_CODE_INVALID_JOB = 204 +EXIT_CODE_INVALID_DATASOURCE = 205 +EXIT_CODE_REFRESH_ERROR = 205 + +EXIT_CODE_FINAL_STATUS_CANCELLED = 210 +EXIT_CODE_FINAL_STATUS_ERRORED = 211 +EXIT_CODE_STATUS_INCOMPLETE = 212 diff --git a/tableau_blueprints/job_status.py b/tableau_blueprints/job_status.py index f6616e5..5ef276b 100644 --- a/tableau_blueprints/job_status.py +++ b/tableau_blueprints/job_status.py @@ -42,7 +42,7 @@ def get_job_info(server, job_id): except Exception as e: print(f'Job {job_id} was not found.') print(e) - sys.exit(errors.EXIT_CODE_JOB_NOT_FOUND) + sys.exit(errors.EXIT_CODE_INVALID_JOB) return job_info @@ -72,7 +72,7 @@ def determine_job_status(server, job_id): exit_code = errors.EXIT_CODE_FINAL_STATUS_CANCELLED else: print(f'Something went wrong when fetching status for job {job_id}') - exit_code = errors.EXIT_CODE_GENERIC_QUERY_JOB_ERROR + exit_code = errors.EXIT_CODE_UNKNOWN_ERROR return exit_code From 13a12a337ffe7781de68b291b1a34a259061801e Mon Sep 17 00:00:00 2001 From: Blake Burch Date: Thu, 9 Jun 2022 21:36:25 -0500 Subject: [PATCH 15/20] Cleaned up code --- tableau_blueprints/authorization.py | 2 +- tableau_blueprints/download_view.py | 77 ++--------------------- tableau_blueprints/job_status.py | 7 +-- tableau_blueprints/lookup.py | 97 +++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 78 deletions(-) create mode 100644 tableau_blueprints/lookup.py diff --git a/tableau_blueprints/authorization.py b/tableau_blueprints/authorization.py index 160b1f1..ada96ca 100644 --- a/tableau_blueprints/authorization.py +++ b/tableau_blueprints/authorization.py @@ -17,7 +17,7 @@ def connect_to_tableau( :param site_id: The site_id for required datasources. ex: ffc7f88a-85a7-48d5-ac03-09ef0a677280 :param server_url: This corresponds to the contentUrl attribute in the Tableau REST API. :param sign_in_method: Whether to log in with username_password or access_token. - :return: connection object + :return: server object, connection object """ if sign_in_method == 'username_password': tableau_auth = TSC.TableauAuth(username, password, site_id=site_id) diff --git a/tableau_blueprints/download_view.py b/tableau_blueprints/download_view.py index 395ee99..310aecc 100644 --- a/tableau_blueprints/download_view.py +++ b/tableau_blueprints/download_view.py @@ -1,7 +1,5 @@ import argparse import sys -import os -import code import shipyard_utils as shipyard import tableauserverclient as TSC @@ -9,9 +7,11 @@ try: import authorization import errors + import lookup except BaseException: from . import authorization from . import errors + from . import lookup def get_args(): @@ -55,72 +55,6 @@ def get_args(): return args -def get_project_id(server, project_name): - """ - Looks up and returns the project_id of the project_name that was specified. - """ - req_option = TSC.RequestOptions() - req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, - TSC.RequestOptions.Operator.Equals, - project_name)) - - project_matches = server.projects.get(req_options=req_option) - if len(project_matches[0]) == 1: - project_id = project_matches[0][0].id - else: - print( - f'{project_name} could not be found. Please check for typos and ensure that the name you provide matches exactly (case sensitive)') - sys.exit(errors.EXIT_CODE_INVALID_PROJECT) - return project_id - - -def get_workbook_id(server, project_id, workbook_name): - """ - Looks up and returns the workbook_id of the workbook_name that was specified, filtered by project_id matches. - """ - req_option = TSC.RequestOptions() - req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, - TSC.RequestOptions.Operator.Equals, - workbook_name)) - - workbook_matches = server.workbooks.get(req_options=req_option) - # We can't filter by project_id in the initial request, - # so we have to find all name matches and look for a project_id match. - workbook_id = None - for workbook in workbook_matches[0]: - if workbook.project_id == project_id: - workbook_id = workbook.id - if workbook_id is None: - print( - f'{workbook_name} could not be found that lives in the project you specified. Please check for typos and ensure that the name(s) you provide match exactly (case sensitive)') - sys.exit(errors.EXIT_CODE_INVALID_WORKBOOK) - return workbook_id - - -def get_view_id(server, project_id, workbook_id, view_name): - """ - Looks up and returns the view_id of the view_name that was specified, filtered by project_id AND workbook_id matches. - """ - req_option = TSC.RequestOptions() - req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, - TSC.RequestOptions.Operator.Equals, - view_name)) - - view_matches = server.views.get(req_options=req_option) - # We can't filter by project_id or workbook_id in the initial request, - # so we have to find all name matches and look for those matches. - view_id = None - for view in view_matches[0]: - if view.project_id == project_id: - if view.workbook_id == workbook_id: - view_id = view.id - if view_id is None: - print( - f'{view_name} could not be found that lives in the project and workbook you specified. Please check for typos and ensure that the name(s) you provide match exactly (case sensitive)') - sys.exit(errors.EXIT_CODE_INVALID_VIEW) - return view_id - - def generate_view_content(server, view_id, file_type): """ Given a specific view_id, populate the view and return the bytes necessary for creating the file. @@ -187,12 +121,13 @@ def main(): sign_in_method) with connection: - project_id = get_project_id(server=server, project_name=project_name) - workbook_id = get_workbook_id( + project_id = lookup.get_project_id( + server=server, project_name=project_name) + workbook_id = lookup.get_workbook_id( server=server, project_id=project_id, workbook_name=workbook_name) - view_id = get_view_id( + view_id = lookup.get_view_id( server=server, project_id=project_id, workbook_id=workbook_id, diff --git a/tableau_blueprints/job_status.py b/tableau_blueprints/job_status.py index 5ef276b..e0b3b9d 100644 --- a/tableau_blueprints/job_status.py +++ b/tableau_blueprints/job_status.py @@ -1,10 +1,7 @@ import tableauserverclient as TSC import argparse import sys -import os -import pickle import shipyard_utils as shipyard -import code try: import errors @@ -100,9 +97,7 @@ def main(): username, password, site_id, server_url, sign_in_method) with connection: - # job_info = get_job_info(server, job_id) - job_status = determine_job_status(server, job_id) - sys.exit(job_status) + sys.exit(determine_job_status(server, job_id)) if __name__ == '__main__': diff --git a/tableau_blueprints/lookup.py b/tableau_blueprints/lookup.py new file mode 100644 index 0000000..b65186e --- /dev/null +++ b/tableau_blueprints/lookup.py @@ -0,0 +1,97 @@ +import sys +import tableauserverclient as TSC + +try: + import errors +except BaseException: + from . import errors + + +def get_project_id(server, project_name): + """ + Looks up and returns the project_id of the project_name that was specified. + """ + req_option = TSC.RequestOptions() + req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, + TSC.RequestOptions.Operator.Equals, + project_name)) + + project_matches = server.projects.get(req_options=req_option) + if len(project_matches[0]) == 1: + project_id = project_matches[0][0].id + else: + print( + f'{project_name} could not be found. Please check for typos and ensure that the name you provide matches exactly (case sensitive)') + sys.exit(errors.EXIT_CODE_INVALID_PROJECT) + return project_id + + +def get_datasource_id(server, project_id, datasource_name): + """ + Looks up and returns the datasource_id of the datasource_name that was specified, filtered by project_id matches. + """ + req_option = TSC.RequestOptions() + req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, + TSC.RequestOptions.Operator.Equals, + datasource_name)) + + datasource_matches = server.datasources.get(req_options=req_option) + + # We can't filter by project_id or project_name in the initial request, + # so we have to find all name matches and look for a project_id match. + datasource_id = None + for datasource in datasource_matches[0]: + if datasource.project_id == project_id: + datasource_id = datasource.id + if datasource_id is None: + print( + f'{datasource_name} could not be found that lives in the project you specified. Please check for typos and ensure that the name(s) you provide match exactly (case sensitive)') + sys.exit(errors.EXIT_CODE_INVALID_DATASOURCE) + return datasource_id + + +def get_workbook_id(server, project_id, workbook_name): + """ + Looks up and returns the workbook_id of the workbook_name that was specified, filtered by project_id matches. + """ + req_option = TSC.RequestOptions() + req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, + TSC.RequestOptions.Operator.Equals, + workbook_name)) + + workbook_matches = server.workbooks.get(req_options=req_option) + # We can't filter by project_id in the initial request, + # so we have to find all name matches and look for a project_id match. + workbook_id = None + for workbook in workbook_matches[0]: + if workbook.project_id == project_id: + workbook_id = workbook.id + if workbook_id is None: + print( + f'{workbook_name} could not be found in the project you specified. Please check for typos and ensure that the name(s) you provide match exactly (case sensitive)') + sys.exit(errors.EXIT_CODE_INVALID_WORKBOOK) + return workbook_id + + +def get_view_id(server, project_id, workbook_id, view_name): + """ + Looks up and returns the view_id of the view_name that was specified, filtered by project_id AND workbook_id matches. + """ + req_option = TSC.RequestOptions() + req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, + TSC.RequestOptions.Operator.Equals, + view_name)) + + view_matches = server.views.get(req_options=req_option) + # We can't filter by project_id or workbook_id in the initial request, + # so we have to find all name matches and look for those matches. + view_id = None + for view in view_matches[0]: + if view.project_id == project_id: + if view.workbook_id == workbook_id: + view_id = view.id + if view_id is None: + print( + f'{view_name} could not be found that lives in the project and workbook you specified. Please check for typos and ensure that the name(s) you provide match exactly (case sensitive)') + sys.exit(errors.EXIT_CODE_INVALID_VIEW) + return view_id From 5aaab9fb828e946ffd5984801ff1d0a256010503 Mon Sep 17 00:00:00 2001 From: Blake Burch Date: Thu, 9 Jun 2022 21:36:48 -0500 Subject: [PATCH 16/20] Added more generic functionality to refresh workbook or datasource. --- ...resh_datasource.py => refresh_resource.py} | 110 ++++++++---------- 1 file changed, 50 insertions(+), 60 deletions(-) rename tableau_blueprints/{refresh_datasource.py => refresh_resource.py} (58%) diff --git a/tableau_blueprints/refresh_datasource.py b/tableau_blueprints/refresh_resource.py similarity index 58% rename from tableau_blueprints/refresh_datasource.py rename to tableau_blueprints/refresh_resource.py index f805e6b..efcddfb 100644 --- a/tableau_blueprints/refresh_datasource.py +++ b/tableau_blueprints/refresh_resource.py @@ -1,19 +1,19 @@ import tableauserverclient as TSC import argparse import sys -import time import shipyard_utils as shipyard -import code # Handle import difference between local and github install try: import job_status import errors import authorization + import lookup except BaseException: from . import job_status from . import errors from . import authorization + from . import lookup def get_args(): @@ -30,10 +30,14 @@ def get_args(): 'username_password', 'access_token'}, required=False) + parser.add_argument( + '--workbook-name', + dest='workbook_name', + required=False) parser.add_argument( '--datasource-name', dest='datasource_name', - required=True) + required=False) parser.add_argument('--project-name', dest='project_name', required=True) parser.add_argument('--check-status', dest='check_status', default='TRUE', choices={ @@ -44,57 +48,9 @@ def get_args(): return args -def get_project_id(server, project_name): - """ - Looks up and returns the project_id of the project_name that was specified. - """ - req_option = TSC.RequestOptions() - req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, - TSC.RequestOptions.Operator.Equals, - project_name)) - - project_matches = server.projects.get(req_options=req_option) - if len(project_matches[0]) == 1: - project_id = project_matches[0][0].id - else: - print( - f'{project_name} could not be found. Please check for typos and ensure that the name you provide matches exactly (case sensitive)') - sys.exit(errors.EXIT_CODE_INVALID_PROJECT) - return project_id - - -def get_datasource_id(server, project_id, datasource_name): - """ - Looks up and returns the datasource_id of the datasource_name that was specified, filtered by project_id matches. - """ - req_option = TSC.RequestOptions() - req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, - TSC.RequestOptions.Operator.Equals, - datasource_name)) - - datasource_matches = server.datasources.get(req_options=req_option) - - # We can't filter by project_id or project_name in the initial request, - # so we have to find all name matches and look for a project_id match. - datasource_id = None - for datasource in datasource_matches[0]: - if datasource.project_id == project_id: - datasource_id = datasource.id - if datasource_id is None: - print( - f'{datasource_name} could not be found that lives in the project you specified. Please check for typos and ensure that the name(s) you provide match exactly (case sensitive)') - sys.exit(errors.EXIT_CODE_INVALID_DATASOURCE) - return datasource_id - - def refresh_datasource(server, datasource_id, datasource_name): - """Refreshes the data of the specified extract. - - :param server: - :param connection: Tableau connection object - :param datasourceobj: The datasource_item specifies the data source to update. - :param datasource_name: The name of the data source. - :return: refreshed id : refreshed object + """ + Refreshes the data of the specified datasource_id. """ try: @@ -115,19 +71,43 @@ def refresh_datasource(server, datasource_id, datasource_name): return refreshed_datasource +def refresh_workbook(server, workbook_id, workbook_name): + """ + Refreshes the data of the specified datasource_id. + """ + + try: + workbook = server.workbooks.get_by_id(workbook_id) + refreshed_workbook = server.workbooks.refresh(workbook) + print(f'Workbook {workbook_name} was successfully triggered.') + except Exception as e: + if 'Resource Conflict' in e.args[0]: + print( + f'A refresh or extract operation for the workbook is already underway.') + if 'is not allowed.' in e.args[0]: + print(f'Refresh or extract operation for the workbook is not allowed.') + else: + print(f'An unknown refresh or extract error occurred.') + print(e) + sys.exit(errors.EXIT_CODE_REFRESH_ERROR) + + return refreshed_datasource + + def main(): args = get_args() username = args.username password = args.password site_id = args.site_id server_url = args.server_url + workbook_name = args.workbook_name datasource_name = args.datasource_name project_name = args.project_name sign_in_method = args.sign_in_method should_check_status = shipyard.args.convert_to_boolean(args.check_status) base_folder_name = shipyard.logs.determine_base_artifact_folder( - 'dbtcloud') + 'tableau') artifact_subfolder_paths = shipyard.logs.determine_artifact_subfolders( base_folder_name) shipyard.logs.create_artifacts_folders(artifact_subfolder_paths) @@ -136,12 +116,22 @@ def main(): username, password, site_id, server_url, sign_in_method) with connection: - project_id = get_project_id(server, project_name) - datasource_id = get_datasource_id( - server, project_id, datasource_name) - refreshed_datasource = refresh_datasource( - server, datasource_id, datasource_name) - job_id = refreshed_datasource.id + project_id = lookup.get_project_id(server, project_name) + + # Allowing user to provide one or the other. These will form two separate Blueprints + # that use the same underlying script. + if datasource_name: + datasource_id = lookup.get_datasource_id( + server, project_id, datasource_name) + refreshed_datasource = refresh_datasource( + server, datasource_id, datasource_name) + job_id = refreshed_datasource.id + if workbook_name: + workbook_id = lookup.get_workbook_id( + server, project_id, workbook_name) + refreshed_workbook = refresh_workbook( + server, workbook_id, workbook_name) + job_id = refreshed_workbook.id if should_check_status: try: From 85d1432fef8e31c4d6d45961d1eaac242bcb32da Mon Sep 17 00:00:00 2001 From: Blake Burch Date: Fri, 10 Jun 2022 16:47:05 -0500 Subject: [PATCH 17/20] Update requirements --- tableau_blueprints/requirements.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tableau_blueprints/requirements.txt b/tableau_blueprints/requirements.txt index 628cf3c..fb20946 100644 --- a/tableau_blueprints/requirements.txt +++ b/tableau_blueprints/requirements.txt @@ -1,5 +1,3 @@ -# List out the required python packages for any of these blueprints to work here. -# Example: -# pandas==1.3.5 -requests==2.27.1 -tableauserverclient==0.18.0 +requests==2.28.0 +tableauserverclient==0.19.0 +shipyard_utils>=0.1.2 \ No newline at end of file From 747c67a097bb58599f1a51ce7a73a4dc405aecf2 Mon Sep 17 00:00:00 2001 From: Blake Burch Date: Fri, 10 Jun 2022 16:48:58 -0500 Subject: [PATCH 18/20] Include logo --- tableau.png | Bin 0 -> 1663 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tableau.png diff --git a/tableau.png b/tableau.png new file mode 100644 index 0000000000000000000000000000000000000000..564502143d9956747c02e24f6519aa44932bc3ce GIT binary patch literal 1663 zcmV-_27vjAP)5E&8(3XuW~?t=uvVUojO9O&P( zAisYg9L#&Sr8Bm!qVCg2nI>oGWsd9(WRQ)3+ak{rDj$}<+!Wj9rR{sOWi2T!Qxn%~ zz2$ze5X-l~DQ|(Sik>T}$?^$$ToGH(HF*PL>><63hL zHUP^OIc}tf*6L+fveJ;|ZSA>J8%~iDSWLncaBU_yWIx-jmk0f>@91_%1&jv?^#W$_ zdcFM8rL2_REC_wOr!oX%#8ezmweRynI?lcoD@Wi#0Zapn5Sq~YqDd+vvh&?ja?mY) z$q0x_Fo7urAc%!@eby)|*wUzBNS*BlQ3@6zDZx@zQu5}Ao|a*mQ*p`UZ)Vd3vcLqE zZvk?FWT-B5frT8}S~)+_`lt*NJF7pdD%|wl3_eN4Zetsb;ZlxJ zWt2tvGsag#jDr&h`o51+K*-@Nk_7A>ZcnRC50XkqcKbBRu$it4Ar9hJfN)A{GzOAe zQB@7tIz;h|i#cA<>s*S0TPt4~h;8HGj0}SoJ!h006xD5v^eONoa`Dk_kg{o{LvI^|s zSv#69#?yXP>F!)gl3b+~SbeC@vk}IK6p$3{CNMeOyjdl3g(bI`h%u*MiH*46n8Ff59GJC1xM>IufXf-Q-f}!EYu&v2-3KNxr2u{M4jwqgfE(g*3XFiwpolm`emW{3qBZ;FlD5;H z(fZ6?zIi>pR}nMF9J(cu!RarzM5U>i-scP& zXpjbcIl2YA3yaMI48-MFA^pvwu)BwL{>Zl|67;DqB>*O{JSFyXUcq#Pp-viT)8fBQ zFB|RMm9N!M49{H7aBt~>1K$7L8oA5293v$csT$7fI<@WI{$;ES!d~PdCZPa}v0MR*+ZwxQF7E%TOVK2skHM?M$wCR> zeVVw8T-q#9izGHhzw>HryY`ds6~jM2I)TNE%RflmEhS|&bmnKtNlNdM(Y(NTVjDq7 zAQaOD=;D9`l28SNo$`z7AfIyhU}BcPjg5_sjg5_sjg8IZ Date: Fri, 10 Jun 2022 17:08:43 -0500 Subject: [PATCH 19/20] Fixed duplicate error code --- tableau_blueprints/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tableau_blueprints/errors.py b/tableau_blueprints/errors.py index b206bf4..59c29cf 100644 --- a/tableau_blueprints/errors.py +++ b/tableau_blueprints/errors.py @@ -9,7 +9,7 @@ EXIT_CODE_INVALID_VIEW = 203 EXIT_CODE_INVALID_JOB = 204 EXIT_CODE_INVALID_DATASOURCE = 205 -EXIT_CODE_REFRESH_ERROR = 205 +EXIT_CODE_REFRESH_ERROR = 206 EXIT_CODE_FINAL_STATUS_CANCELLED = 210 EXIT_CODE_FINAL_STATUS_ERRORED = 211 From 5b8280f1992af4ee442d146133dbeb09212ee315 Mon Sep 17 00:00:00 2001 From: Blake Burch Date: Fri, 10 Jun 2022 17:13:47 -0500 Subject: [PATCH 20/20] Fixed artifact folder from dbt to tableau --- tableau_blueprints/job_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tableau_blueprints/job_status.py b/tableau_blueprints/job_status.py index e0b3b9d..d8b37b8 100644 --- a/tableau_blueprints/job_status.py +++ b/tableau_blueprints/job_status.py @@ -82,7 +82,7 @@ def main(): sign_in_method = args.sign_in_method base_folder_name = shipyard.logs.determine_base_artifact_folder( - 'dbtcloud') + 'tableau') artifact_subfolder_paths = shipyard.logs.determine_artifact_subfolders( base_folder_name) shipyard.logs.create_artifacts_folders(artifact_subfolder_paths)