diff --git a/python-lib/sharepoint_client.py b/python-lib/sharepoint_client.py index 7c9e5a1..8d0c010 100644 --- a/python-lib/sharepoint_client.py +++ b/python-lib/sharepoint_client.py @@ -7,6 +7,7 @@ import time import json import re +import calendar from xml.etree.ElementTree import Element, tostring from xml.dom import minidom @@ -421,6 +422,40 @@ def get_list_items(self, list_title, params=None): self.assert_response_ok(response, calling_method="get_list_items") return response.json().get("ListData", {}) + def get_list_last_modified(self, list_title, params=None): + params = params or {} + data = { + "parameters": { + "__metadata": { + "type": "SP.RenderListDataParameters" + }, + "RenderOptions": SharePointConstants.RENDER_OPTIONS, + "AllowMultipleValueFilterForTaxonomyFields": True, + "AddRequiredFields": True + } + } + headers = DSSConstants.JSON_HEADERS + response = self.session.post( + self.get_list_data_as_stream(list_title), + params=params, + headers=headers, + json=data + ) + self.assert_response_ok(response, calling_method="get_list_last_modified") + last_item_modified_time = response.json().get("lastItemModifiedTime") + last_item_modified_time_epoch = 0 + if last_item_modified_time: + try: + last_item_modified_time_epoch = calendar.timegm( + time.strptime(last_item_modified_time, "%Y-%m-%d %H:%M:%SZ") + ) * 1000 + except Exception as error: + logger.error("Could not convert lastItemModifiedTime ({}): {}".format( + last_item_modified_time, + error + )) + return last_item_modified_time_epoch + def create_list(self, list_name): headers = DSSConstants.JSON_HEADERS data = { diff --git a/python-triggers/check-list-modified/trigger.json b/python-triggers/check-list-modified/trigger.json new file mode 100644 index 0000000..003ff8a --- /dev/null +++ b/python-triggers/check-list-modified/trigger.json @@ -0,0 +1,183 @@ +{ + "meta": { + "label": "Check modified list", + "description": "Check that a SharePoint list has been modified", + "icon": "icon-cloud" + }, + "params": [ + { + "name": "auth_type", + "label": "Type of authentication", + "type": "SELECT", + "selectChoices": [ + { + "value": "login", + "label": "User name / password (deprecated)" + }, + { + "value": "oauth", + "label": "Azure Single Sign On" + }, + { + "value": "site-app-permissions", + "label": "Site App Permissions" + }, + { + "value": "app-certificate", + "label": "Certificates" + }, + { + "value": "app-username-password", + "label": "User name / password" + } + ] + }, + { + "name": "sharepoint_oauth", + "label": "Azure preset", + "type": "PRESET", + "parameterSetId": "oauth-login", + "visibilityCondition": "model.auth_type == 'oauth'" + }, + { + "name": "sharepoint_sharepy", + "label": "SharePoint preset", + "type": "PRESET", + "parameterSetId": "sharepoint-login", + "visibilityCondition": "model.auth_type == 'login'" + }, + { + "name": "site_app_permissions", + "label": "Site App preset", + "type": "PRESET", + "parameterSetId": "site-app-permissions", + "visibilityCondition": "model.auth_type == 'site-app-permissions'" + }, + { + "name": "app_certificate", + "label": "Certificates", + "type": "PRESET", + "parameterSetId": "app-certificate", + "visibilityCondition": "model.auth_type == 'app-certificate'" + }, + { + "name": "app_username_password", + "label": "App username password", + "type": "PRESET", + "parameterSetId": "app-username-password", + "visibilityCondition": "model.auth_type == 'app-username-password'" + }, + { + "name": "sharepoint_list_title", + "label": "List title", + "defaultValue": "DSS_${projectKey}_", + "description": "", + "type": "STRING", + "mandatory": true + }, + { + "name": "expand_lookup", + "label": "Expand lookup fields", + "description": "", + "type": "BOOLEAN", + "defaultValue": false, + "visibilityCondition": false, + "mandatory": true + }, + { + "name": "metadata_to_retrieve", + "label": "Metadata to retrieve", + "type": "MULTISELECT", + "visibilityCondition": false, + "selectChoices": [ + { + "value": "ID", + "label": "Item's ID" + }, + { + "value": "Created", + "label": "Creation date" + }, + { + "value": "Modified", + "label": "Modification date" + }, + { + "value": "Author", + "label": "Created by" + }, + { + "value": "Editor", + "label": "Modified by" + } + ] + }, + { + "name": "advanced_parameters", + "label": "Show advanced parameters", + "description": "", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "name": "sharepoint_site_overwrite", + "label": "Site path preset overwrite", + "type": "STRING", + "description": "sites/site_name/subsite...", + "visibilityCondition": "model.advanced_parameters == true" + }, + { + "name": "sharepoint_list_view_title", + "label": "View name", + "description": "Read data from a specific view", + "type": "STRING", + "defaultValue": "", + "visibilityCondition": "model.advanced_parameters == true" + }, + { + "name": "write_mode", + "label": "Write mode", + "type": "SELECT", + "defaultValue": "create", + "selectChoices": [ + { + "value": "create", + "label": "Create a new list" + }, + { + "value": "append", + "label": "Append to existing list" + } + ], + "visibilityCondition": false + }, + { + "name": "max_workers", + "label": "Max nb of workers (write mode only)", + "description": "More workers will speed writing but also randomize items order", + "visibilityCondition": "model.advanced_parameters == true", + "type": "INT", + "defaultValue": 1, + "minI": 1, + "maxI": 5 + }, + { + "name": "batch_size", + "label": "Batch size (write mode only)", + "description": "Number of items writen per batch", + "visibilityCondition": "model.advanced_parameters == true", + "type": "INT", + "defaultValue": 100, + "minI": 1, + "maxI": 100 + }, + { + "name": "attempt_session_reset_on_403", + "label": "Attempt session reset", + "description": "Slow, refer to documentation", + "type": "BOOLEAN", + "defaultValue": false, + "visibilityCondition": "model.advanced_parameters == true" + } + ] +} \ No newline at end of file diff --git a/python-triggers/check-list-modified/trigger.py b/python-triggers/check-list-modified/trigger.py new file mode 100644 index 0000000..99292f3 --- /dev/null +++ b/python-triggers/check-list-modified/trigger.py @@ -0,0 +1,75 @@ +import time +import logging +from dataiku.customtrigger import get_plugin_config +from dataiku.scenario import Trigger +from sharepoint_client import SharePointClient +from dss_constants import DSSConstants + + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO, + format='sharepoint-online plugin %(levelname)s - %(message)s') + + +class ProjectVariable(): + def __init__(self, variable_name, variable_type=None, default_value=None): + from dataiku import Project + self.variable_name = variable_name + self.project = Project() + self.variable_type = variable_type or "standard" + self.default_value = default_value + + def is_not_set(self): + project_variables = self.project.get_variables() + project_variable = project_variables.get(self.variable_type, {}).get(self.variable_name) + if project_variable is None: + return True + return False + + def get_value(self): + project_variables = self.project.get_variables() + project_variable = project_variables.get(self.variable_type, {}).get(self.variable_name, self.default_value) + return project_variable + + def set_value(self, value): + project_variables = self.project.get_variables() + project_variables[self.variable_type][self.variable_name] = value + self.project.set_variables(project_variables) + + +def pretty_epoch(epoch_time): + return time.strftime('%Y-%m-%d %H:%M:%S%z', time.gmtime(epoch_time/1000)) + + +logger.info('SharePoint Online plugin list trigger v{}'.format(DSSConstants.PLUGIN_VERSION)) +plugin_config = get_plugin_config() +config = plugin_config.get("config", {}) +sharepoint_list_title = config.get("sharepoint_list_title") + +trigger = Trigger() +project_variable_name = "sharepoint-online-list-trigger_{}".format(sharepoint_list_title) +last_modified = ProjectVariable(project_variable_name, default_value=0) +client = SharePointClient(config) +remote_file_last_modified_epoch = client.get_list_last_modified( + sharepoint_list_title +) +last_modified_epoch = last_modified.get_value() +logger.info("Trigger.{}.lastLocalTime: {} ({})".format( + project_variable_name, + last_modified_epoch, + pretty_epoch(last_modified_epoch) + ) +) +logger.info("Trigger.{}.remoteTime: {} ({})".format( + project_variable_name, + remote_file_last_modified_epoch, + pretty_epoch(remote_file_last_modified_epoch) + ) +) +if last_modified.is_not_set() or (remote_file_last_modified_epoch > last_modified_epoch): + logger.info("remote epoch {} > local epoch {}, firing the trigger".format(remote_file_last_modified_epoch, last_modified_epoch)) + remote_file_last_modified_epoch = int(time.time()) * 1000 + last_modified.set_value(remote_file_last_modified_epoch) + trigger.fire() +else: + logger.info("Remote spreadsheet has not been modified")