From 4d2e15b64a52f6f00de48fe50cb72ea470f54f03 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 4 Sep 2025 16:16:34 +0200 Subject: [PATCH 1/2] add SP fs trigger based on lastModified --- .../check-file-modified/trigger.json | 98 ++++++++++++++++ .../check-file-modified/trigger.py | 109 ++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 python-triggers/check-file-modified/trigger.json create mode 100644 python-triggers/check-file-modified/trigger.py diff --git a/python-triggers/check-file-modified/trigger.json b/python-triggers/check-file-modified/trigger.json new file mode 100644 index 0000000..f4568e7 --- /dev/null +++ b/python-triggers/check-file-modified/trigger.json @@ -0,0 +1,98 @@ +{ + "meta": { + "label": "Check file/folder was modified", + "description": "Check that a SharePoint file or folder has been modified", + "icon": "icon-cloud" + }, + "params": [ + { + "name": "auth_type", + "label": "Type of authentication", + "type": "SELECT", + "selectChoices": [ + { + "value": "login", + "label": "User name / password (deprecated)" + }, + { + "value": "oauth", + "label": "Azure Single Sign On" + }, + { + "value": "site-app-permissions", + "label": "Site App Permissions" + }, + { + "value": "app-certificate", + "label": "Certificates" + }, + { + "value": "app-username-password", + "label": "User name / password" + } + ] + }, + { + "name": "sharepoint_oauth", + "label": "Azure preset", + "type": "PRESET", + "parameterSetId": "oauth-login", + "visibilityCondition": "model.auth_type == 'oauth'" + }, + { + "name": "sharepoint_sharepy", + "label": "SharePoint preset", + "type": "PRESET", + "parameterSetId": "sharepoint-login", + "visibilityCondition": "model.auth_type == 'login'" + }, + { + "name": "site_app_permissions", + "label": "Site App preset", + "type": "PRESET", + "parameterSetId": "site-app-permissions", + "visibilityCondition": "model.auth_type == 'site-app-permissions'" + }, + { + "name": "app_certificate", + "label": "Certificates", + "type": "PRESET", + "parameterSetId": "app-certificate", + "visibilityCondition": "model.auth_type == 'app-certificate'" + }, + { + "name": "app_username_password", + "label": "App username password", + "type": "PRESET", + "parameterSetId": "app-username-password", + "visibilityCondition": "model.auth_type == 'app-username-password'" + }, + { + "name": "advanced_parameters", + "label": "Show advanced parameters", + "description": "", + "type": "BOOLEAN", + "defaultValue": false + }, + { + "name": "sharepoint_site_overwrite", + "label": "Site path preset overwrite", + "type": "STRING", + "description": "sites/site_name/subsite...", + "visibilityCondition": "model.advanced_parameters == true" + }, + { + "name": "sharepoint_root_overwrite", + "label": "Root directory preset overwrite", + "type": "STRING", + "description": "", + "visibilityCondition": "model.advanced_parameters == true" + }, + { + "name": "sharepoint_path", + "label": "File path", + "type": "STRING", + "description": "" + } + ] +} \ No newline at end of file diff --git a/python-triggers/check-file-modified/trigger.py b/python-triggers/check-file-modified/trigger.py new file mode 100644 index 0000000..6cc4827 --- /dev/null +++ b/python-triggers/check-file-modified/trigger.py @@ -0,0 +1,109 @@ +import time +import logging +import calendar +from dataiku.customtrigger import get_plugin_config +from dataiku.scenario import Trigger +from sharepoint_client import SharePointClient +from dss_constants import DSSConstants + + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO, + format='sharepoint-online plugin %(levelname)s - %(message)s') + + +class ProjectVariable(): + def __init__(self, variable_name, variable_type=None, default_value=None): + from dataiku import Project + self.variable_name = variable_name + self.project = Project() + self.variable_type = variable_type or "standard" + self.default_value = default_value + + def is_not_set(self): + project_variables = self.project.get_variables() + project_variable = project_variables.get(self.variable_type, {}).get(self.variable_name) + if project_variable is None: + return True + return False + + def get_value(self): + project_variables = self.project.get_variables() + project_variable = project_variables.get(self.variable_type, {}).get(self.variable_name, self.default_value) + return project_variable + + def set_value(self, value): + project_variables = self.project.get_variables() + project_variables[self.variable_type][self.variable_name] = value + self.project.set_variables(project_variables) + + +def pretty_epoch(epoch_time): + return time.strftime('%Y-%m-%d %H:%M:%S%z', time.gmtime(epoch_time/1000)) + + +def find_sharepoint_item(sharepoint_path): + sharepoint_path = sharepoint_path.strip("/") + sharepoint_path = "/" + sharepoint_path + sharepoint_path_tokens = sharepoint_path.split("/") + item_path = "/".join(sharepoint_path_tokens[:-1]) + item_name = "/".join(sharepoint_path_tokens[-1:]) + + # 1 - get files folders for path-1 + # 2 - scan these files then folders for Name=path[last] !!files get priority + # 3 - returns TimeLastModified. Format is 2024-01-18T14:41:46Z + files = client.get_files(item_path) + files = files.get("d", {}).get("results", []) + # print("ALX:files={}".format(files)) + for file in files: + file_name = file.get("Name") + if file_name == item_name: + return file + folders = client.get_folders(item_path) + folders = folders.get("d", {}).get("results", []) + for folder in folders: + folder_name = folder.get("Name") + if folder_name == item_name: + return folder + + +logger.info('SharePoint Online plugin fs trigger v{}'.format(DSSConstants.PLUGIN_VERSION)) +plugin_config = get_plugin_config() +config = plugin_config.get("config", {}) +sharepoint_path = config.get("sharepoint_path") + +trigger = Trigger() +project_variable_name = "sharepoint-online-fs-trigger_{}".format(sharepoint_path) +last_modified = ProjectVariable(project_variable_name, default_value=0) +client = SharePointClient(config) + +sharepoint_item = find_sharepoint_item(sharepoint_path) +if not sharepoint_item: + raise Exception("Sharepoint item not found") + +remote_file_last_modified = sharepoint_item.get("TimeLastModified") + +remote_file_last_modified_epoch = calendar.timegm( + time.strptime(remote_file_last_modified, "%Y-%m-%dT%H:%M:%SZ") + ) * 1000 + +last_modified_epoch = last_modified.get_value() +logger.info("Trigger.{}.lastLocalTime: {} ({})".format( + project_variable_name, + last_modified_epoch, + pretty_epoch(last_modified_epoch) + ) +) +logger.info("Trigger.{}.remoteTime: {} ({})".format( + project_variable_name, + remote_file_last_modified_epoch, + pretty_epoch(remote_file_last_modified_epoch) + ) +) +if last_modified.is_not_set() or (remote_file_last_modified_epoch > last_modified_epoch): + logger.info("remote epoch {} > local epoch {}, firing the trigger".format(remote_file_last_modified_epoch, last_modified_epoch)) + remote_file_last_modified_epoch = int(time.time()) * 1000 + last_modified.set_value(remote_file_last_modified_epoch) + trigger.fire() +else: + logger.info("Remote spreadsheet has not been modified") From e63541160c6b141cd37345d4bf7546218a3900eb Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Thu, 4 Sep 2025 16:51:19 +0200 Subject: [PATCH 2/2] change in message --- python-triggers/check-file-modified/trigger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-triggers/check-file-modified/trigger.py b/python-triggers/check-file-modified/trigger.py index 6cc4827..cc65f31 100644 --- a/python-triggers/check-file-modified/trigger.py +++ b/python-triggers/check-file-modified/trigger.py @@ -106,4 +106,4 @@ def find_sharepoint_item(sharepoint_path): last_modified.set_value(remote_file_last_modified_epoch) trigger.fire() else: - logger.info("Remote spreadsheet has not been modified") + logger.info("Remote file has not been modified")