From 4aacf2ec766f620166af92bfc0710c0282427499 Mon Sep 17 00:00:00 2001 From: Kevin Jump Date: Wed, 17 Apr 2024 12:27:01 +0100 Subject: [PATCH] From v13: Fixes (#590) - missing null check --- .../Extensions/uSyncActionExtensions.cs | 12 +- .../SyncHandlers/SyncHandlerRoot.cs | 4334 +++++++++-------- 2 files changed, 2179 insertions(+), 2167 deletions(-) diff --git a/uSync.BackOffice/Extensions/uSyncActionExtensions.cs b/uSync.BackOffice/Extensions/uSyncActionExtensions.cs index 4024b5c3..7f93c813 100644 --- a/uSync.BackOffice/Extensions/uSyncActionExtensions.cs +++ b/uSync.BackOffice/Extensions/uSyncActionExtensions.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using Umbraco.Extensions; @@ -61,4 +62,13 @@ public static IEnumerable ConvertToSummary(this IEnumerable + /// try to find an action in the list based on key, and handler alias + /// + public static bool TryFindAction(this IEnumerable actions, Guid key, string handlerAlias, out uSyncAction action) + { + action = actions.FirstOrDefault(x => $"{x.key}_{x.HandlerAlias}" == $"{key}_{handlerAlias}", new uSyncAction { key = Guid.Empty }); + return action.key != Guid.Empty; + } + } diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs index 6e137f16..3118170d 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerRoot.cs @@ -36,2170 +36,2172 @@ namespace uSync.BackOffice.SyncHandlers; /// public abstract class SyncHandlerRoot { - /// - /// Reference to the Logger - /// - protected readonly ILogger> logger; - - /// - /// Reference to the uSyncFileService - /// - protected readonly SyncFileService syncFileService; - - /// - /// Reference to the Event service used to handle locking - /// - protected readonly uSyncEventService _mutexService; - - /// - /// List of dependency checkers for this Handler - /// - protected readonly IList> dependencyCheckers; - - /// - /// List of change trackers for this handler - /// - protected readonly IList> trackers; - - /// - /// The Serializer to use for importing/exporting items - /// - protected ISyncSerializer serializer; - - /// - /// Runtime cache for caching lookups - /// - protected readonly IAppPolicyCache runtimeCache; - - /// - /// Alias of the handler, used when getting settings from the configuration file - /// - public string Alias { get; private set; } - - /// - /// Name of handler, displayed to user during reports/imports - /// - public string Name { get; private set; } - - /// - /// name of the folder inside the uSync folder where items are stored - /// - public string DefaultFolder { get; private set; } - - /// - /// priority order items are imported in - /// - /// - /// to import before anything else go below USYNC_RESERVED_LOWER (1000) - /// to import after uSync has done all the other things go past USYNC_RESERVED_UPPER (2000) - /// - public int Priority { get; private set; } - - /// - /// Icon displayed on the screen while the import happens. - /// - public string Icon { get; private set; } - - /// - /// does this handler require two passes at the import (e.g data-types import once, and then again after doc-types) - /// - protected bool IsTwoPass = false; - - /// - /// the object type of the item being processed. - /// - public string ItemType { get; protected set; } = typeof(TObject).Name; - - /// - /// Is the handler enabled - /// - public bool Enabled { get; set; } = true; - - - /// - /// the default configuration for this handler - /// - public HandlerSettings DefaultConfig { get; set; } - - /// - /// the root folder for the handler (based on the settings) - /// - [Obsolete("we should be using the array of folders, will be removed in v15")] - protected string rootFolder { get; set; } - - /// - /// the root folders to use for the handler (based on settings). - /// - protected string[] rootFolders { get; set; } - - /// - /// the UDIEntityType for the handler objects - /// - public string EntityType { get; protected set; } - - /// - /// Name of the type (object) - /// - public string TypeName { get; protected set; } // we calculate these now based on the entityType ? - - /// - /// UmbracoObjectType of items handled by this handler - /// - protected UmbracoObjectTypes itemObjectType { get; set; } = UmbracoObjectTypes.Unknown; - - /// - /// UmbracoObjectType of containers manged by this handler - /// - protected UmbracoObjectTypes itemContainerType = UmbracoObjectTypes.Unknown; - - /// - /// The type of the handler - /// - protected string handlerType; - - /// - /// SyncItem factory reference - /// - protected readonly ISyncItemFactory itemFactory; - - /// - /// Reference to the uSyncConfigService - /// - protected readonly uSyncConfigService uSyncConfig; - - /// - /// Umbraco's shortStringHelper - /// - protected readonly IShortStringHelper shortStringHelper; - - /// - /// Constructor, base for all handlers - /// - public SyncHandlerRoot( - ILogger> logger, - AppCaches appCaches, - IShortStringHelper shortStringHelper, - SyncFileService syncFileService, - uSyncEventService mutexService, - uSyncConfigService uSyncConfig, - ISyncItemFactory itemFactory) - { - this.uSyncConfig = uSyncConfig; - - this.logger = logger; - this.shortStringHelper = shortStringHelper; - this.itemFactory = itemFactory; - - var _serializer = this.itemFactory.GetSerializers().FirstOrDefault(); - if (_serializer is null) - throw new KeyNotFoundException($"No Serializer found for handler {this.Alias}"); - - this.serializer = _serializer; - this.trackers = this.itemFactory.GetTrackers().ToList(); - this.dependencyCheckers = this.itemFactory.GetCheckers().ToList(); - - - this.syncFileService = syncFileService; - this._mutexService = mutexService; - - var currentHandlerType = GetType(); - var meta = currentHandlerType.GetCustomAttribute(false); - if (meta == null) - throw new InvalidOperationException($"The Handler {handlerType} requires a {typeof(SyncHandlerAttribute)}"); - - handlerType = currentHandlerType.ToString(); - Name = meta.Name; - Alias = meta.Alias; - DefaultFolder = meta.Folder; - Priority = meta.Priority; - IsTwoPass = meta.IsTwoPass; - Icon = string.IsNullOrWhiteSpace(meta.Icon) ? "icon-umb-content" : meta.Icon; - EntityType = meta.EntityType; - - TypeName = serializer.ItemType; - - this.itemObjectType = uSyncObjectType.ToUmbracoObjectType(EntityType); - this.itemContainerType = uSyncObjectType.ToContainerUmbracoObjectType(EntityType); - - this.DefaultConfig = GetDefaultConfig(); - rootFolder = uSyncConfig.GetRootFolder(); - rootFolders = uSyncConfig.GetFolders(); - - if (uSyncConfig.Settings.CacheFolderKeys) - { - this.runtimeCache = appCaches.RuntimeCache; - } - else - { - logger.LogInformation("No caching of handler key lookups (CacheFolderKeys = false)"); - this.runtimeCache = NoAppCache.Instance; - } - } - - private HandlerSettings GetDefaultConfig() - { - var defaultSet = uSyncConfig.GetDefaultSetSettings(); - var config = defaultSet.GetHandlerSettings(this.Alias); - - if (defaultSet.DisabledHandlers.InvariantContains(this.Alias)) - config.Enabled = false; - - return config; - } - - #region Importing - - /// - /// Import everything from a given folder, using the supplied configuration settings. - /// - public IEnumerable ImportAll(string folder, HandlerSettings config, bool force, SyncUpdateCallback? callback = null) - => ImportAll([folder], config, new uSyncImportOptions - { - Flags = force ? SerializerFlags.Force : SerializerFlags.None, - Callbacks = new uSyncCallbacks(null, callback) - }); - - /// - /// import everything from a collection of folders, using the supplied config. - /// - /// - /// allows us to 'merge' a collection of folders down and perform an import against them (without first having to actually merge the folders on disk) - /// - public IEnumerable ImportAll(string[] folders, HandlerSettings config, uSyncImportOptions options) - { - var cacheKey = PrepCaches(); - runtimeCache.ClearByKey(cacheKey); - - options.Callbacks?.Update?.Invoke("Calculating import order", 1, 9); - - var items = GetMergedItems(folders); - - options.Callbacks?.Update?.Invoke($"Processing {items.Count} items", 2, 9); - - // create the update list with items.count space. this is the max size we need this list. - List actions = new List(items.Count); - List> updates = new List>(items.Count); - List cleanMarkers = []; - - int count = 0; - int total = items.Count; - - foreach (var item in items) - { - count++; - - options.Callbacks?.Update?.Invoke($"Importing {Path.GetFileNameWithoutExtension(item.Path)}", count, total); - - var result = ImportElement(item.Node, item.Path, config, options); - foreach (var attempt in result) - { - if (attempt.Success) - { - if (attempt.Change == ChangeType.Clean) - { - cleanMarkers.Add(item.Path); - } - else if (attempt.Item is not null && attempt.Item is TObject update) - { - updates.Add(new ImportedItem(item.Node, update)); - } - } - - if (attempt.Change != ChangeType.Clean) - actions.Add(attempt); - } - } - - // clean up memory we didn't use in the update list. - updates.TrimExcess(); - - // bulk save? - if (updates.Count > 0) - { - if (options.Flags.HasFlag(SerializerFlags.DoNotSave)) - { - serializer.Save(updates.Select(x => x.Item)); - } - - PerformSecondPassImports(updates, actions, config, options.Callbacks?.Update); - } - - if (actions.All(x => x.Success) && cleanMarkers.Count > 0) - { - PerformImportClean(cleanMarkers, actions, config, options.Callbacks?.Update); - } - - CleanCaches(cacheKey); - options.Callbacks?.Update?.Invoke("Done", 3, 3); - - return actions; - } - - /// - /// get all items for the report/import process. - /// - /// - /// - public IReadOnlyList FetchAllNodes(string[] folders) - => GetMergedItems(folders); - - /// - /// method to get the merged folders, handlers that care about orders should override this. - /// - protected virtual IReadOnlyList GetMergedItems(string[] folders) - { - var baseTracker = trackers.FirstOrDefault() as ISyncTrackerBase; - return syncFileService.MergeFolders(folders, uSyncConfig.Settings.DefaultExtension, baseTracker).ToArray(); - } - - private void PerformImportClean(List cleanMarkers, List actions, HandlerSettings config, SyncUpdateCallback? callback) - { - foreach (var item in cleanMarkers.Select((filePath, Index) => new { filePath, Index })) - { - var folderName = Path.GetFileName(item.filePath); - callback?.Invoke($"Cleaning {folderName}", item.Index, cleanMarkers.Count); - - var cleanActions = CleanFolder(item.filePath, false, config.UseFlatStructure); - if (cleanActions.Any()) - { - actions.AddRange(cleanActions); - } - else - { - // nothing to delete, we report this as a no change - actions.Add(uSyncAction.SetAction( - success: true, - name: $"Folder {Path.GetFileName(item.filePath)}", - change: ChangeType.NoChange, - filename: syncFileService.GetSiteRelativePath(item.filePath))); - } - } - // remove the actual cleans (they will have been replaced by the deletes - actions.RemoveAll(x => x.Change == ChangeType.Clean); - } - - /// - /// Import everything in a given (child) folder, based on setting - /// - [Obsolete("Import folder method not called directly from v13.1 will be removed in v15")] - protected virtual IEnumerable ImportFolder(string folder, HandlerSettings config, Dictionary updates, bool force, SyncUpdateCallback? callback) - { - List actions = new List(); - var files = GetImportFiles(folder); - - var flags = SerializerFlags.None; - if (force) flags |= SerializerFlags.Force; - - var cleanMarkers = new List(); - - int count = 0; - int total = files.Count(); - foreach (string file in files) - { - count++; - - callback?.Invoke($"Importing {Path.GetFileNameWithoutExtension(file)}", count, total); - - var result = Import(file, config, flags); - foreach (var attempt in result) - { - if (attempt.Success) - { - if (attempt.Change == ChangeType.Clean) - { - cleanMarkers.Add(file); - } - else if (attempt.Item != null && attempt.Item is TObject item) - { - updates.Add(file, item); - } - } - - if (attempt.Change != ChangeType.Clean) - actions.Add(attempt); - } - } - - // bulk save .. - if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any()) - { - // callback?.Invoke($"Saving {updates.Count()} changes", 1, 1); - serializer.Save(updates.Select(x => x.Value)); - } - - var folders = syncFileService.GetDirectories(folder); - foreach (var children in folders) - { - actions.AddRange(ImportFolder(children, config, updates, force, callback)); - } - - if (actions.All(x => x.Success) && cleanMarkers.Count > 0) - { - foreach (var item in cleanMarkers.Select((filePath, Index) => new { filePath, Index })) - { - var folderName = Path.GetFileName(item.filePath); - callback?.Invoke($"Cleaning {folderName}", item.Index, cleanMarkers.Count); - - var cleanActions = CleanFolder(item.filePath, false, config.UseFlatStructure); - if (cleanActions.Any()) - { - actions.AddRange(cleanActions); - } - else - { - // nothing to delete, we report this as a no change - actions.Add(uSyncAction.SetAction( - success: true, - name: $"Folder {Path.GetFileName(item.filePath)}", - change: ChangeType.NoChange, filename: syncFileService.GetSiteRelativePath(item.filePath) - ) - ); - } - } - // remove the actual cleans (they will have been replaced by the deletes - actions.RemoveAll(x => x.Change == ChangeType.Clean); - } - - return actions; - } - - /// - /// Import a single item, from the .config file supplied - /// - public virtual IEnumerable Import(string filePath, HandlerSettings config, SerializerFlags flags) - { - try - { - syncFileService.EnsureFileExists(filePath); - var node = syncFileService.LoadXElement(filePath); - return Import(node, filePath, config, flags); - } - catch (FileNotFoundException notFoundException) - { - return uSyncAction.Fail(Path.GetFileName(filePath), this.handlerType, this.ItemType, ChangeType.Fail, $"File not found {notFoundException.Message}", notFoundException) - .AsEnumerableOfOne(); - } - catch (Exception ex) - { - logger.LogWarning("{alias}: Import Failed : {exception}", this.Alias, ex.ToString()); - return uSyncAction.Fail(Path.GetFileName(filePath), this.handlerType, this.ItemType, ChangeType.Fail, $"Import Fail: {ex.Message}", new Exception(ex.Message, ex)) - .AsEnumerableOfOne(); - } - } - - /// - /// Import a single item based on already loaded XML - /// - public virtual IEnumerable Import(XElement node, string filename, HandlerSettings config, SerializerFlags flags) - { - if (config.FailOnMissingParent) flags |= SerializerFlags.FailMissingParent; - return ImportElement(node, filename, config, new uSyncImportOptions { Flags = flags }); - } - - /// - /// Import a single item from a usync XML file - /// - virtual public IEnumerable Import(string file, HandlerSettings config, bool force) - { - var flags = SerializerFlags.OnePass; - if (force) flags |= SerializerFlags.Force; - - return Import(file, config, flags); - } - - /// - /// Import a node, with settings and options - /// - /// - /// All Imports lead here - /// - virtual public IEnumerable ImportElement(XElement node, string filename, HandlerSettings settings, uSyncImportOptions options) - { - if (!ShouldImport(node, settings)) - { - return uSyncAction.SetAction(true, node.GetAlias(), message: "Change blocked (based on configuration)") - .AsEnumerableOfOne(); - } - - if (_mutexService.FireItemStartingEvent(new uSyncImportingItemNotification(node, (ISyncHandler)this))) - { - // blocked - return uSyncActionHelper - .ReportAction(ChangeType.NoChange, node.GetAlias(), node.GetPath(), GetNameFromFileOrNode(filename, node), node.GetKey(), this.Alias, "Change stopped by delegate event") - .AsEnumerableOfOne(); - } - - try - { - // merge the options from the handler and any import options into our serializer options. - var serializerOptions = new SyncSerializerOptions(options.Flags, settings.Settings, options.UserId); - serializerOptions.MergeSettings(options.Settings); - - // get the item. - var attempt = DeserializeItem(node, serializerOptions); - var action = uSyncActionHelper.SetAction(attempt, GetNameFromFileOrNode(filename, node), node.GetKey(), this.Alias, IsTwoPass); - - // add item if we have it. - if (attempt.Item != null) action.Item = attempt.Item; - - // add details if we have them - if (attempt.Details != null && attempt.Details.Any()) action.Details = attempt.Details; - - // this might not be the place to do this because, two pass items are imported at another point too. - _mutexService.FireItemCompletedEvent(new uSyncImportedItemNotification(node, attempt.Change)); - - - return action.AsEnumerableOfOne(); - } - catch (Exception ex) - { - logger.LogWarning("{alias}: Import Failed : {exception}", this.Alias, ex.ToString()); - return uSyncAction.Fail(Path.GetFileName(filename), this.handlerType, this.ItemType, ChangeType.Fail, - $"{this.Alias} Import Fail: {ex.Message}", new Exception(ex.Message)) - .AsEnumerableOfOne(); - } - - } - - - /// - /// Works through a list of items that have been processed and performs the second import pass on them. - /// - private void PerformSecondPassImports(List> importedItems, List actions, HandlerSettings config, SyncUpdateCallback? callback = null) - { - foreach (var item in importedItems.Select((update, Index) => new { update, Index })) - { - var itemKey = item.update.Node.GetKey(); - - callback?.Invoke($"Second Pass {item.update.Node.GetKey()}", item.Index, importedItems.Count); - var attempt = ImportSecondPass(item.update.Node, item.update.Item, config, callback); - if (attempt.Success) - { - // if the second attempt has a message on it, add it to the first attempt. - if (!string.IsNullOrWhiteSpace(attempt.Message) || attempt.Details?.Any() == true) - { - uSyncAction action = actions.FirstOrDefault(x => $"{x.key}_{x.HandlerAlias}" == $"{itemKey}_{this.Alias}", new uSyncAction { key = Guid.Empty }); - if (action.key != Guid.Empty) - { - actions.Remove(action); - action.Message += attempt.Message ?? ""; - - if (attempt.Details?.Any() == true) - { - var details = action.Details?.ToList() ?? []; - details.AddRange(attempt.Details); - action.Details = details; - } - actions.Add(action); - } - } - if (attempt.Change > ChangeType.NoChange && !attempt.Saved && attempt.Item != null) - { - serializer.Save(attempt.Item.AsEnumerableOfOne()); - } - } - else - { - uSyncAction action = actions.FirstOrDefault(x => $"{x.key}_{x.HandlerAlias}" == $"{itemKey}_{this.Alias}", new uSyncAction { key = Guid.Empty }); - if (action.key != Guid.Empty) - { - actions.Remove(action); - action.Success = attempt.Success; - action.Message = $"Second Pass Fail: {attempt.Message}"; - action.Exception = attempt.Exception; - actions.Add(action); - } - } - } - } - - /// - /// Perform a second pass import on an item - /// - virtual public IEnumerable ImportSecondPass(uSyncAction action, HandlerSettings settings, uSyncImportOptions options) - { - if (!IsTwoPass) return Enumerable.Empty(); - - try - { - var fileName = action.FileName; - - if (fileName is null || syncFileService.FileExists(fileName) is false) - return Enumerable.Empty(); - - var node = syncFileService.LoadXElement(fileName); - var item = GetFromService(node.GetKey()); - if (item == null) return Enumerable.Empty(); - - // merge the options from the handler and any import options into our serializer options. - var serializerOptions = new SyncSerializerOptions(options?.Flags ?? SerializerFlags.None, settings.Settings, options?.UserId ?? -1); - serializerOptions.MergeSettings(options?.Settings); - - // do the second pass on this item - var result = DeserializeItemSecondPass(item, node, serializerOptions); - - return uSyncActionHelper.SetAction(result, syncFileService.GetSiteRelativePath(fileName), node.GetKey(), this.Alias).AsEnumerableOfOne(); - } - catch (Exception ex) - { - logger.LogWarning($"Second Import Failed: {ex}"); - return uSyncAction.Fail(action.Name, this.handlerType, action.ItemType, ChangeType.ImportFail, "Second import failed", ex).AsEnumerableOfOne(); - } - } - - - /// - /// Perform a 'second pass' import on a single item. - /// - [Obsolete("Call method with node element to reduce disk IO, will be removed in v15")] - virtual public SyncAttempt ImportSecondPass(string file, TObject item, HandlerSettings config, SyncUpdateCallback callback) - { - if (IsTwoPass) - { - try - { - syncFileService.EnsureFileExists(file); - - var flags = SerializerFlags.None; - - var node = syncFileService.LoadXElement(file); - return DeserializeItemSecondPass(item, node, new SyncSerializerOptions(flags, config.Settings)); - } - catch (Exception ex) - { - logger.LogWarning($"Second Import Failed: {ex.ToString()}"); - return SyncAttempt.Fail(GetItemAlias(item), item, ChangeType.Fail, ex.Message, ex); - } - } - - return SyncAttempt.Succeed(GetItemAlias(item), ChangeType.NoChange); - } - - /// - /// Perform a 'second pass' import on a single item. - /// - virtual public SyncAttempt ImportSecondPass(XElement node, TObject item, HandlerSettings config, SyncUpdateCallback? callback) - { - if (IsTwoPass is false) - return SyncAttempt.Succeed(GetItemAlias(item), ChangeType.NoChange); - - try - { - return DeserializeItemSecondPass(item, node, new SyncSerializerOptions(SerializerFlags.None, config.Settings)); - } - catch (Exception ex) - { - logger.LogWarning($"Second Import Failed: {ex.ToString()}"); - return SyncAttempt.Fail(GetItemAlias(item), item, ChangeType.Fail, ex.Message, ex); - } - } - - /// - /// given a folder we calculate what items we can remove, because they are - /// not in one the files in the folder. - /// - protected virtual IEnumerable CleanFolder(string cleanFile, bool reportOnly, bool flat) - { - var folder = Path.GetDirectoryName(cleanFile); - if (string.IsNullOrWhiteSpace(folder) is true || syncFileService.DirectoryExists(folder) is false) - return Enumerable.Empty(); - - // get the keys for every item in this folder. - - // this would works on the flat folder structure too, - // there we are being super defensive, so if an item - // is anywhere in the folder it won't get removed - // even if the folder is wrong - // be a little slower (not much though) - - // we cache this, (it is cleared on an ImportAll) - var keys = GetFolderKeys(folder, flat); - if (keys.Count > 0) - { - // move parent to here, we only need to check it if there are files. - var parent = GetCleanParent(cleanFile); - if (parent == null) return Enumerable.Empty(); - - logger.LogDebug("Got parent with {alias} from clean file {file}", GetItemAlias(parent), Path.GetFileName(cleanFile)); - - // keys should aways have at least one entry (the key from cleanFile) - // if it doesn't then something might have gone wrong. - // because we are being defensive when it comes to deletes, - // we only then do deletes when we know we have loaded some keys! - return DeleteMissingItems(parent, keys, reportOnly); - } - else - { - logger.LogWarning("Failed to get the keys for items in the folder, there might be a disk issue {folder}", folder); - return Enumerable.Empty(); - } - } - - /// - /// pre-populates the cache folder key list. - /// - /// - /// this means if we are calling the process multiple times, - /// we can optimise the key code and only load it once. - /// - public void PreCacheFolderKeys(string folder, IList folderKeys) - { - var cacheKey = $"{GetCacheKeyBase()}_{folder.GetHashCode()}"; - runtimeCache.ClearByKey(cacheKey); - runtimeCache.GetCacheItem(cacheKey, () => folderKeys); - } - - /// - /// Get the GUIDs for all items in a folder - /// - /// - /// This is disk intensive, (checking the .config files all the time) - /// so we cache it, and if we are using the flat folder structure, then - /// we only do it once, so its quicker. - /// - protected IList GetFolderKeys(string folder, bool flat) - { - // We only need to load all the keys once per handler (if all items are in a folder that key will be used). - var folderKey = folder.GetHashCode(); - - var cacheKey = $"{GetCacheKeyBase()}_{folderKey}"; - - - return runtimeCache.GetCacheItem(cacheKey, () => - { - logger.LogDebug("Getting Folder Keys : {cacheKey}", cacheKey); - - // when it's not flat structure we also get the sub folders. (extra defensive get them all) - var keys = new List(); - var files = syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}", !flat).ToList(); - - foreach (var file in files) - { - var node = XElement.Load(file); - var key = node.GetKey(); - if (key != Guid.Empty && !keys.Contains(key)) - { - keys.Add(key); - } - } - - logger.LogDebug("Loaded {count} keys from {folder} [{cacheKey}]", keys.Count, folder, cacheKey); - - return keys; - - }, null) ?? []; - } - - /// - /// Get the parent item of the clean file (so we can check if the folder has any versions of this item in it) - /// - protected TObject? GetCleanParent(string file) - { - var node = XElement.Load(file); - var key = node.GetKey(); - if (key == Guid.Empty) return default; - return GetFromService(key); - } - - /// - /// remove an items that are not listed in the GUIDs to keep - /// - /// parent item that all keys will be under - /// list of GUIDs of items we don't want to delete - /// will just report what would happen (doesn't do the delete) - /// list of delete actions - protected abstract IEnumerable DeleteMissingItems(TObject parent, IEnumerable keysToKeep, bool reportOnly); - - /// - /// Remove an items that are not listed in the GUIDs to keep. - /// - /// parent item that all keys will be under - /// list of GUIDs of items we don't want to delete - /// will just report what would happen (doesn't do the delete) - /// list of delete actions - protected virtual IEnumerable DeleteMissingItems(int parentId, IEnumerable keysToKeep, bool reportOnly) - => Enumerable.Empty(); - - /// - /// Get the files we are going to import from a folder. - /// - protected virtual IEnumerable GetImportFiles(string folder) - => syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}").OrderBy(x => x); - - /// - /// check to see if this element should be imported as part of the process. - /// - virtual protected bool ShouldImport(XElement node, HandlerSettings config) - { - // if createOnly is on, then we only create things that are not already there. - // this lookup is slow (relatively) so we only do it if we have to. - if (config.GetSetting(Core.uSyncConstants.DefaultSettings.CreateOnly, Core.uSyncConstants.DefaultSettings.CreateOnly_Default) - || config.GetSetting(Core.uSyncConstants.DefaultSettings.OneWay, Core.uSyncConstants.DefaultSettings.CreateOnly_Default)) - { - var item = serializer.FindItem(node); - if (item != null) - { - logger.LogDebug("CreateOnly: Item {alias} already exist not importing it.", node.GetAlias()); - return false; - } - } - - - // Ignore alias setting. - // if its set the thing with this alias is ignored. - var ignore = config.GetSetting("IgnoreAliases", string.Empty); - if (!string.IsNullOrWhiteSpace(ignore)) - { - var ignoreList = ignore.ToDelimitedList(); - if (ignoreList.InvariantContains(node.GetAlias())) - { - logger.LogDebug("Ignore: Item {alias} is in the ignore list", node.GetAlias()); - return false; - } - } - - - return true; - } - - - /// - /// Check to see if this element should be exported. - /// - virtual protected bool ShouldExport(XElement node, HandlerSettings config) => true; - - #endregion - - #region Exporting - - /// - /// Export all items to a give folder on the disk - /// - virtual public IEnumerable ExportAll(string folder, HandlerSettings config, SyncUpdateCallback? callback) - => ExportAll([folder], config, callback); - - /// - /// Export all items to a give folder on the disk - /// - virtual public IEnumerable ExportAll(string[] folders, HandlerSettings config, SyncUpdateCallback? callback) - { - // we don't clean the folder out on an export all. - // because the actions (renames/deletes) live in the folder - // - // there will have to be a different clean option - // syncFileService.CleanFolder(folder); - - return ExportAll(default, folders, config, callback); - } - - /// - /// export all items underneath a given container - /// - virtual public IEnumerable ExportAll(TContainer? parent, string folder, HandlerSettings config, SyncUpdateCallback? callback) - => ExportAll(parent, [folder], config, callback); - - /// - /// Export all items to a give folder on the disk - /// - virtual public IEnumerable ExportAll(TContainer? parent, string[] folders, HandlerSettings config, SyncUpdateCallback? callback) - { - var actions = new List(); - - if (itemContainerType != UmbracoObjectTypes.Unknown) - { - var containers = GetFolders(parent); - foreach (var container in containers) - { - actions.AddRange(ExportAll(container, folders, config, callback)); - } - } - - var items = GetChildItems(parent).ToList(); - foreach (var item in items.Select((Value, Index) => new { Value, Index })) - { - TObject? concreteType; - if (item.Value is TObject t) - { - concreteType = t; - } - else - { - concreteType = GetFromService(item.Value); - } - if (concreteType is not null) - { // only export the items (not the containers). - callback?.Invoke(GetItemName(concreteType), item.Index, items.Count); - actions.AddRange(Export(concreteType, folders, config)); - } - actions.AddRange(ExportAll(item.Value, folders, config, callback)); - } - - return actions; - } - - /// - /// Fetch all child items beneath a given container - /// - abstract protected IEnumerable GetChildItems(TContainer? parent); - - /// - /// Fetch all child items beneath a given folder - /// - /// - /// - abstract protected IEnumerable GetFolders(TContainer? parent); - - /// - /// Does this container have any children - /// - public bool HasChildren(TContainer item) - => GetFolders(item).Any() || GetChildItems(item).Any(); - - - /// - /// Export a single item based on it's ID - /// - public IEnumerable Export(int id, string folder, HandlerSettings settings) - => Export(id, [folder], settings); - - /// - /// Export an item based on its id, observing root behavior. - /// - public IEnumerable Export(int id, string[] folders, HandlerSettings settings) - { - var item = this.GetFromService(id); - if (item is null) - { - return uSyncAction.Fail( - id.ToString(), this.handlerType, this.ItemType, - ChangeType.Export, "Unable to find item", - new KeyNotFoundException($"Item of {id} cannot be found")) - .AsEnumerableOfOne(); - } - return this.Export(item, folders, settings); - } - - /// - /// Export an single item from a given UDI value - /// - public IEnumerable Export(Udi udi, string folder, HandlerSettings settings) - => Export(udi, [folder], settings); - - /// - /// Export an single item from a given UDI value - /// - public IEnumerable Export(Udi udi, string[] folders, HandlerSettings settings) - { - var item = FindByUdi(udi); - if (item != null) - return Export(item, folders, settings); - - return uSyncAction.Fail(nameof(udi), this.handlerType, this.ItemType, ChangeType.Fail, $"Item not found {udi}", - new KeyNotFoundException(nameof(udi))) - .AsEnumerableOfOne(); - } - - /// - /// Export a given item to disk - /// - virtual public IEnumerable Export(TObject item, string folder, HandlerSettings config) - => Export(item, [folder], config); - - /// - /// Export a given item to disk - /// - virtual public IEnumerable Export(TObject item, string[] folders, HandlerSettings config) - { - if (item == null) - return uSyncAction.Fail(nameof(item), this.handlerType, this.ItemType, ChangeType.Fail, "Item not set", - new ArgumentNullException(nameof(item))).AsEnumerableOfOne(); - - if (_mutexService.FireItemStartingEvent(new uSyncExportingItemNotification(item, (ISyncHandler)this))) - { - return uSyncActionHelper - .ReportAction(ChangeType.NoChange, GetItemName(item), string.Empty, string.Empty, GetItemKey(item), this.Alias, - "Change stopped by delegate event") - .AsEnumerableOfOne(); - } - - var targetFolder = folders.Last(); - - var filename = GetPath(targetFolder, item, config.GuidNames, config.UseFlatStructure) - .ToAppSafeFileName(); - - // - if (IsLockedAtRoot(folders, filename.Substring(targetFolder.Length + 1))) - { - // if we have lock roots on, then this item will not export - // because exporting would mean the root was no longer used. - return uSyncAction.SetAction(true, syncFileService.GetSiteRelativePath(filename), - type: typeof(TObject).ToString(), - change: ChangeType.NoChange, - message: "Not exported (would overwrite root value)", - filename: filename).AsEnumerableOfOne(); - } - - - var attempt = Export_DoExport(item, filename, folders, config); - - if (attempt.Change > ChangeType.NoChange) - _mutexService.FireItemCompletedEvent(new uSyncExportedItemNotification(attempt.Item, ChangeType.Export)); - - return uSyncActionHelper.SetAction(attempt, syncFileService.GetSiteRelativePath(filename), GetItemKey(item), this.Alias).AsEnumerableOfOne(); - } - - /// - /// Do the meat of the export - /// - /// - /// inheriting this method, means you don't have to repeat all the checks in child handlers. - /// - protected virtual SyncAttempt Export_DoExport(TObject item, string filename, string[] folders, HandlerSettings config) - { - var attempt = SerializeItem(item, new SyncSerializerOptions(config.Settings)); - if (attempt.Success && attempt.Item is not null) - { - if (ShouldExport(attempt.Item, config)) - { - // only write the file to disk if it should be exported. - syncFileService.SaveXElement(attempt.Item, filename); - - if (config.CreateClean && HasChildren(item)) - { - CreateCleanFile(GetItemKey(item), filename); - } - } - else - { - return SyncAttempt.Succeed(Path.GetFileName(filename), ChangeType.NoChange, "Not Exported (Based on configuration)"); - } - } - - return attempt; - } - - /// - /// Checks to see if this item is locked at the root level (meaning we shouldn't save it) - /// - protected bool IsLockedAtRoot(string[] folders, string path) - { - if (folders.Length <= 1) return false; - - if (ExistsInFolders(folders[..^1], path.TrimStart(['\\', '/']))) - { - return uSyncConfig.Settings.LockRoot || uSyncConfig.Settings.LockRootTypes.InvariantContains(EntityType); - } - - return false; - - bool ExistsInFolders(string[] folders, string path) - { - foreach (var folder in folders) - { - if (syncFileService.FileExists(Path.Combine(folder, path.TrimStart(['\\', '/'])))) - { - return true; - } - } - - return false; - } - } - - /// - /// does this item have any children ? - /// - /// - /// on items where we can check this (quickly) we can reduce the number of checks we might - /// make on child items or cleaning up where we don't need to. - /// - protected virtual bool HasChildren(TObject item) - => true; - - /// - /// create a clean file, which is used as a marker, when performing remote deletes. - /// - protected void CreateCleanFile(Guid key, string filename) - { - if (string.IsNullOrWhiteSpace(filename) || key == Guid.Empty) - return; - - var folder = Path.GetDirectoryName(filename); - var name = Path.GetFileNameWithoutExtension(filename); - - if (string.IsNullOrEmpty(folder)) return; - - var cleanPath = Path.Combine(folder, $"{name}_clean.config"); - - var node = XElementExtensions.MakeEmpty(key, SyncActionType.Clean, $"clean {name} children"); - node.Add(new XAttribute("itemType", serializer.ItemType)); - syncFileService.SaveXElement(node, cleanPath); - } - - #endregion - - #region Reporting - - /// - /// Run a report based on a given folder - /// - public IEnumerable Report(string folder, HandlerSettings config, SyncUpdateCallback? callback) - => Report([folder], config, callback); - - /// - /// Run a report based on a set of folders. - /// - public IEnumerable Report(string[] folders, HandlerSettings config, SyncUpdateCallback? callback) - { - List actions = []; - - var cacheKey = PrepCaches(); - - callback?.Invoke("Organising import structure", 1, 3); - - var items = GetMergedItems(folders); - - int count = 0; - - foreach (var item in items) - { - count++; - callback?.Invoke(Path.GetFileNameWithoutExtension(item.Path), count, items.Count); - actions.AddRange(ReportElement(item.Node, item.FileName, config)); - } - - callback?.Invoke("Validating Report", 2, 3); - var validationActions = ReportMissingParents(actions.ToArray()); - actions.AddRange(ReportDeleteCheck(uSyncConfig.GetRootFolder(), validationActions)); - - CleanCaches(cacheKey); - callback?.Invoke("Done", 3, 3); - return actions; - } - - private List ValidateReport(string folder, List actions) - { - // Alters the existing list, by changing the type as needed. - var validationActions = ReportMissingParents(actions.ToArray()); - - // adds new actions - for delete clashes. - validationActions.AddRange(ReportDeleteCheck(folder, validationActions)); - - return validationActions; - } - - /// - /// Check to returned report to see if there is a delete and an update for the same item - /// because if there is then we have issues. - /// - protected virtual IEnumerable ReportDeleteCheck(string folder, IEnumerable actions) - { - var duplicates = new List(); - - // delete checks. - foreach (var deleteAction in actions.Where(x => x.Change != ChangeType.NoChange && x.Change == ChangeType.Delete)) - { - // todo: this is only matching by key, but non-tree based serializers also delete by alias. - // so this check actually has to be booted back down to the serializer. - if (actions.Any(x => x.Change != ChangeType.Delete && DoActionsMatch(x, deleteAction))) - { - var duplicateAction = uSyncActionHelper.ReportActionFail(deleteAction.Name, - $"Duplicate! {deleteAction.Name} exists both as delete and import action"); - - // create a detail message to tell people what has happened. - duplicateAction.DetailMessage = "uSync detected a duplicate actions, where am item will be both created and deleted."; - var details = new List(); - - // add the delete message to the list of changes - var filename = Path.GetFileName(deleteAction.FileName) ?? string.Empty; - var relativePath = deleteAction.FileName?.Substring(folder.Length) ?? string.Empty; - - details.Add(uSyncChange.Delete(filename, $"Delete: {deleteAction.Name} ({filename}", relativePath)); - - // add all the duplicates to the list of changes. - foreach (var dup in actions.Where(x => x.Change != ChangeType.Delete && DoActionsMatch(x, deleteAction))) - { - var dupFilename = Path.GetFileName(dup.FileName) ?? string.Empty; - var dupRelativePath = dup.FileName?.Substring(folder.Length) ?? string.Empty; - - details.Add( - uSyncChange.Update( - path: dupFilename, - name: $"{dup.Change} : {dup.Name} ({dupFilename})", - oldValue: "", - newValue: dupRelativePath)); - } - - duplicateAction.Details = details; - duplicates.Add(duplicateAction); - } - } - - return duplicates; - } - - /// - /// check to see if an action matches, another action. - /// - /// - /// how two actions match can vary based on handler, in the most part they are matched by key - /// but some items will also check based on the name. - /// - /// when we are dealing with handlers where things can have the same - /// name (tree items, such as content or media), this function has - /// to be overridden to remove the name check. - /// - protected virtual bool DoActionsMatch(uSyncAction a, uSyncAction b) - { - if (a.key == b.key) return true; - if (a.Name.Equals(b.Name, StringComparison.InvariantCultureIgnoreCase)) return true; - return false; - } - - /// - /// check if a node matches a item - /// - /// - /// Like above we want to match on key and alias, but only if the alias is unique. - /// however the GetItemAlias function is overridden by tree based handlers to return a unique - /// alias (the key again), so we don't get false positives. - /// - protected virtual bool DoItemsMatch(XElement node, TObject item) - { - if (GetItemKey(item) == node.GetKey()) return true; - - // yes this is an or, we've done it explicitly, so you can tell! - if (node.GetAlias().Equals(this.GetItemAlias(item), StringComparison.InvariantCultureIgnoreCase)) return true; - - return false; - } - - /// - /// Check report for any items that are missing their parent items - /// - /// - /// The serializers will report if an item is missing a parent item within umbraco, - /// but because the serializer isn't aware of the wider import (all the other items) - /// it can't say if the parent is in the import. - /// - /// This method checks for the parent of an item in the wider list of items being - /// imported. - /// - private List ReportMissingParents(uSyncAction[] actions) - { - for (int i = 0; i < actions.Length; i++) - { - if (actions[i].Change != ChangeType.ParentMissing || actions[i].FileName is null) continue; - - var node = XElement.Load(actions[i].FileName!); - var guid = node.GetParentKey(); - - if (guid != Guid.Empty) - { - if (actions.Any(x => x.key == guid && (x.Change < ChangeType.Fail || x.Change == ChangeType.ParentMissing))) - { - logger.LogDebug("Found existing key in actions {item}", actions[i].Name); - actions[i].Change = ChangeType.Create; - } - else - { - logger.LogWarning("{item} is missing a parent", actions[i].Name); - } - } - } - - return actions.ToList(); - } - - /// - /// Run a report on a given folder - /// - public virtual IEnumerable ReportFolder(string folder, HandlerSettings config, SyncUpdateCallback? callback) - { - - List actions = new List(); - - var files = GetImportFiles(folder).ToList(); - - int count = 0; - - logger.LogDebug("ReportFolder: {folder} ({count} files)", folder, files.Count); - - foreach (string file in files) - { - count++; - callback?.Invoke(Path.GetFileNameWithoutExtension(file), count, files.Count); - - actions.AddRange(ReportItem(file, config)); - } - - foreach (var children in syncFileService.GetDirectories(folder)) - { - actions.AddRange(ReportFolder(children, config, callback)); - } - - return actions; - } - - /// - /// Report on any changes for a single XML node. - /// - protected virtual IEnumerable ReportElement(XElement node, string filename, HandlerSettings? config) - => ReportElement(node, filename, config ?? this.DefaultConfig, new uSyncImportOptions()); - - - /// - /// Report an Element - /// - public IEnumerable ReportElement(XElement node, string filename, HandlerSettings settings, uSyncImportOptions options) - { - try - { - // starting reporting notification - // this lets us intercept a report and - // shortcut the checking (sometimes). - if (_mutexService.FireItemStartingEvent(new uSyncReportingItemNotification(node))) - { - return uSyncActionHelper - .ReportAction(ChangeType.NoChange, node.GetAlias(), node.GetPath(), GetNameFromFileOrNode(filename, node), node.GetKey(), this.Alias, - "Change stopped by delegate event") - .AsEnumerableOfOne(); - } - - var actions = new List(); - - // get the serializer options - var serializerOptions = new SyncSerializerOptions(options.Flags, settings.Settings, options.UserId); - serializerOptions.MergeSettings(options.Settings); - - // check if this item is current (the provided XML and exported XML match) - var change = IsItemCurrent(node, serializerOptions); - - var action = uSyncActionHelper - .ReportAction(change.Change, node.GetAlias(), node.GetPath(), GetNameFromFileOrNode(filename, node), node.GetKey(), this.Alias, ""); - - - - action.Message = ""; - - if (action.Change == ChangeType.Clean) - { - actions.AddRange(CleanFolder(filename, true, settings.UseFlatStructure)); - } - else if (action.Change > ChangeType.NoChange) - { - if (change.CurrentNode is not null) - { - action.Details = GetChanges(node, change.CurrentNode, serializerOptions); - if (action.Change != ChangeType.Create && (action.Details == null || action.Details.Count() == 0)) - { - action.Message = "XML is different - but properties may not have changed"; - action.Details = MakeRawChange(node, change.CurrentNode, serializerOptions).AsEnumerableOfOne(); - } - else - { - action.Message = $"{action.Change}"; - } - } - actions.Add(action); - } - else - { - actions.Add(action); - } - - // tell other things we have reported this item. - _mutexService.FireItemCompletedEvent(new uSyncReportedItemNotification(node, action.Change)); - - return actions; - } - catch (FormatException fex) - { - return uSyncActionHelper - .ReportActionFail(Path.GetFileName(node.GetAlias()), $"format error {fex.Message}") - .AsEnumerableOfOne(); - } - } - - private uSyncChange MakeRawChange(XElement node, XElement current, SyncSerializerOptions options) - { - if (current != null) - return uSyncChange.Update(node.GetAlias(), "Raw XML", current.ToString(), node.ToString()); - - return uSyncChange.NoChange(node.GetAlias(), node.GetAlias()); - } - - /// - /// Run a report on a single file. - /// - protected IEnumerable ReportItem(string file, HandlerSettings config) - { - try - { - var node = syncFileService.LoadXElement(file); - - if (ShouldImport(node, config)) - { - return ReportElement(node, file, config); - } - else - { - return uSyncActionHelper.ReportAction(ChangeType.NoChange, node.GetAlias(), node.GetPath(), syncFileService.GetSiteRelativePath(file), node.GetKey(), - this.Alias, "Will not be imported (Based on configuration)") - .AsEnumerableOfOne(); - } - } - catch (Exception ex) - { - return uSyncActionHelper - .ReportActionFail(Path.GetFileName(file), $"Reporting error {ex.Message}") - .AsEnumerableOfOne(); - } - - } - - - private IEnumerable GetChanges(XElement node, XElement currentNode, SyncSerializerOptions options) - => itemFactory.GetChanges(node, currentNode, options); - - #endregion - - #region Notification Events - - /// - /// calculate if this handler should process the events. - /// - /// - /// will check if uSync is paused, the handler is enabled or the action is set. - /// - protected bool ShouldProcessEvent() - { - if (_mutexService.IsPaused) return false; - if (!DefaultConfig.Enabled) return false; - - - var group = !string.IsNullOrWhiteSpace(DefaultConfig.Group) ? DefaultConfig.Group : this.Group; - - if (uSyncConfig.Settings.ExportOnSave.InvariantContains("All") || - uSyncConfig.Settings.ExportOnSave.InvariantContains(group)) - { - return HandlerActions.Save.IsValidAction(DefaultConfig.Actions); - } - - return false; - } - - /// - /// Handle an Umbraco Delete notification - /// - public virtual void Handle(DeletedNotification notification) - { - if (!ShouldProcessEvent()) return; - - foreach (var item in notification.DeletedEntities) - { - try - { - var handlerFolders = GetDefaultHandlerFolders(); - ExportDeletedItem(item, handlerFolders, DefaultConfig); - } - catch (Exception ex) - { - logger.LogWarning(ex, "Failed to create delete marker"); - notification.Messages.Add(new EventMessage("uSync", $"Failed to mark as deleted : {ex.Message}", EventMessageType.Warning)); - } - } - } - - /// - /// Handle the Umbraco Saved notification for items. - /// - /// - public virtual void Handle(SavedNotification notification) - { - if (!ShouldProcessEvent()) return; - if (notification.State.TryGetValue(uSync.EventPausedKey, out var paused) && paused is true) - return; - - var handlerFolders = GetDefaultHandlerFolders(); - - foreach (var item in notification.SavedEntities) - { - try - { - var attempts = Export(item, handlerFolders, DefaultConfig); - foreach (var attempt in attempts.Where(x => x.Success)) - { - if (attempt.FileName is null) continue; - this.CleanUp(item, attempt.FileName, handlerFolders.Last()); - } - } - catch (Exception ex) - { - logger.LogWarning(ex, "Failed to create uSync export file"); - notification.Messages.Add(new EventMessage("uSync", $"Failed to create export file : {ex.Message}", EventMessageType.Warning)); - } - } - } - - /// - /// Handle the Umbraco moved notification for items. - /// - /// - public virtual void Handle(MovedNotification notification) - { - try - { - if (!ShouldProcessEvent()) return; - HandleMove(notification.MoveInfoCollection); - } - catch (Exception ex) - { - logger.LogWarning(ex, "Failed to export move operation"); - notification.Messages.Add(new EventMessage("uSync", $"Failed to export move : {ex.Message}", EventMessageType.Warning)); - } - - } - - /// - /// Process a collection of move events - /// - /// - /// This has been separated out, because we also call this code when a handler supports - /// recycle bin events - /// - protected void HandleMove(IEnumerable> moveInfoCollection) - { - foreach (var item in moveInfoCollection) - { - var handlerFolders = GetDefaultHandlerFolders(); - var attempts = Export(item.Entity, handlerFolders, DefaultConfig); - - if (!this.DefaultConfig.UseFlatStructure) - { - // moves only need cleaning up if we are not using flat, because - // with flat the file will always be in the same folder. - - foreach (var attempt in attempts.Where(x => x.Success is true)) - { - if (attempt.FileName is null) continue; - this.CleanUp(item.Entity, attempt.FileName, handlerFolders.Last()); - } - } - } - } - - /// - /// Export any deletes items to disk - /// - /// - /// Deleted items get 'empty' files on disk so we know they where deleted - /// - protected virtual void ExportDeletedItem(TObject item, string folder, HandlerSettings config) - => ExportDeletedItem(item, [folder], config); - - /// - /// Export any deletes items to disk - /// - /// - /// Deleted items get 'empty' files on disk so we know they where deleted - /// - protected virtual void ExportDeletedItem(TObject item, string[] folders, HandlerSettings config) - { - if (item == null) return; - - var targetFolder = folders.Last(); - - var filename = GetPath(targetFolder, item, config.GuidNames, config.UseFlatStructure) - .ToAppSafeFileName(); - - if (IsLockedAtRoot(folders, filename.Substring(targetFolder.Length + 1))) - { - // don't do anything this thing exists at a higher level. ! - return; - } - - - var attempt = serializer.SerializeEmpty(item, SyncActionType.Delete, string.Empty); - if (attempt.Item is not null && ShouldExport(attempt.Item, config) is true) - { - if (attempt.Success && attempt.Change != ChangeType.NoChange) - { - syncFileService.SaveXElement(attempt.Item, filename); - - // so check - it shouldn't (under normal operation) - // be possible for a clash to exist at delete, because nothing else - // will have changed (like name or location) - - // we only then do this if we are not using flat structure. - if (!DefaultConfig.UseFlatStructure) - this.CleanUp(item, filename, Path.Combine(folders.Last(), this.DefaultFolder)); - } - } - } - - /// - /// get all the possible folders for this handlers - /// - protected string[] GetDefaultHandlerFolders() - => rootFolders.Select(f => Path.Combine(f, DefaultFolder)).ToArray(); - - - /// - /// Cleans up the handler folder, removing duplicate files for this item - /// - /// - /// e.g if someone renames a thing (and we are using the name in the file) - /// this will clean anything else in the folder that has that key / alias - /// - protected virtual void CleanUp(TObject item, string newFile, string folder) - { - var physicalFile = syncFileService.GetAbsPath(newFile); - - var files = syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}"); - - foreach (string file in files) - { - // compare the file paths. - if (!syncFileService.PathMatches(physicalFile, file)) // This is not the same file, as we are saving. - { - try - { - var node = syncFileService.LoadXElement(file); - - // if this XML file matches the item we have just saved. - - if (!node.IsEmptyItem() || node.GetEmptyAction() != SyncActionType.Rename) - { - // the node isn't empty, or its not a rename (because all clashes become renames) - - if (DoItemsMatch(node, item)) - { - logger.LogDebug("Duplicate {file} of {alias}, saving as rename", Path.GetFileName(file), this.GetItemAlias(item)); - - var attempt = serializer.SerializeEmpty(item, SyncActionType.Rename, node.GetAlias()); - if (attempt.Success && attempt.Item is not null) - { - syncFileService.SaveXElement(attempt.Item, file); - } - } - } - } - catch (Exception ex) - { - logger.LogWarning("Error during cleanup of existing files {message}", ex.Message); - // cleanup should fail silently ? - because it can impact on normal Umbraco operations? - } - } - } - - var folders = syncFileService.GetDirectories(folder); - foreach (var children in folders) - { - CleanUp(item, newFile, children); - } - } - - #endregion - - // 98% of the time the serializer can do all these calls for us, - // but for blueprints, we want to get different items, (but still use the - // content serializer) so we override them. - - - /// - /// Fetch an item via the Serializer - /// - protected virtual TObject? GetFromService(int id) => serializer.FindItem(id); - - /// - /// Fetch an item via the Serializer - /// - protected virtual TObject? GetFromService(Guid key) => serializer.FindItem(key); - - /// - /// Fetch an item via the Serializer - /// - protected virtual TObject? GetFromService(string alias) => serializer.FindItem(alias); - - /// - /// Delete an item via the Serializer - /// - protected virtual void DeleteViaService(TObject item) => serializer.DeleteItem(item); - - /// - /// Get the alias of an item from the Serializer - /// - protected string GetItemAlias(TObject item) => serializer.ItemAlias(item); - - /// - /// Get the Key of an item from the Serializer - /// - protected Guid GetItemKey(TObject item) => serializer.ItemKey(item); - - /// - /// Get a container item from the Umbraco service. - /// - abstract protected TObject? GetFromService(TContainer? item); - - /// - /// Get a container item from the Umbraco service. - /// - virtual protected TContainer? GetContainer(Guid key) => default; - - /// - /// Get a container item from the Umbraco service. - /// - virtual protected TContainer? GetContainer(int id) => default; - - /// - /// Get the file path to use for an item - /// - /// Item to derive path for - /// should we use the key value in the path - /// should the file be flat and ignore any sub folders? - /// relative file path to use for an item - virtual protected string GetItemPath(TObject item, bool useGuid, bool isFlat) - => useGuid ? GetItemKey(item).ToString() : GetItemFileName(item); - - /// - /// Get the file name to use for an item - /// - virtual protected string GetItemFileName(TObject item) - => GetItemAlias(item).ToSafeFileName(shortStringHelper); - - /// - /// Get the name of a supplied item - /// - abstract protected string GetItemName(TObject item); - - /// - /// Calculate the relative Physical path value for any item - /// - /// - /// this is where a item is saved on disk in relation to the uSync folder - /// - virtual protected string GetPath(string folder, TObject item, bool GuidNames, bool isFlat) - { - if (isFlat && GuidNames) return Path.Combine(folder, $"{GetItemKey(item)}.{this.uSyncConfig.Settings.DefaultExtension}"); - var path = Path.Combine(folder, $"{this.GetItemPath(item, GuidNames, isFlat)}.{this.uSyncConfig.Settings.DefaultExtension}"); - - // if this is flat but not using GUID filenames, then we check for clashes. - if (isFlat && !GuidNames) return CheckAndFixFileClash(path, item); - return path; - } - - - /// - /// Get a clean filename that doesn't clash with any existing items. - /// - /// - /// clashes we want to resolve can occur when the safeFilename for an item - /// matches with the safe file name for something else. e.g - /// 1 Special Doc-type - /// 2 Special Doc-type - /// - /// Will both resolve to SpecialDocType.Config - /// - /// the first item to be written to disk for a clash will get the 'normal' name - /// all subsequent items will get the appended name. - /// - /// this can be completely sidestepped by using GUID filenames. - /// - virtual protected string CheckAndFixFileClash(string path, TObject item) - { - if (syncFileService.FileExists(path)) - { - var node = syncFileService.LoadXElement(path); - - if (node == null) return path; - if (GetItemKey(item) == node.GetKey()) return path; - if (GetXmlMatchString(node) == GetItemMatchString(item)) return path; - - // get here we have a clash, we should append something - var append = GetItemKey(item).ToShortKeyString(8); // (this is the shortened GUID like media folders do) - return Path.Combine(Path.GetDirectoryName(path) ?? string.Empty, - Path.GetFileNameWithoutExtension(path) + "_" + append + Path.GetExtension(path)); - } - - return path; - } - - /// - /// a string we use to match this item, with other (where there are levels) - /// - /// - /// this works because unless it's content/media you can't actually have - /// clashing aliases at different levels in the folder structure. - /// - /// So just checking the alias works, for content we overwrite these two functions. - /// - protected virtual string GetItemMatchString(TObject item) => GetItemAlias(item); - - /// - /// Calculate the matching item string from the loaded uSync XML element - /// - protected virtual string GetXmlMatchString(XElement node) => node.GetAlias(); - - /// - /// Rename an item - /// - /// - /// This doesn't get called, because renames generally are handled in the serialization because we match by key. - /// - virtual public uSyncAction Rename(TObject item) => new uSyncAction(); - - - /// - /// Group a handler belongs too (default will be settings) - /// - public virtual string Group { get; protected set; } = uSyncConstants.Groups.Settings; - - /// - /// Serialize an item to XML based on a given UDI value - /// - public SyncAttempt GetElement(Udi udi) - { - var element = FindByUdi(udi); - if (element != null) - return SerializeItem(element, new SyncSerializerOptions()); - - return SyncAttempt.Fail(udi.ToString(), ChangeType.Fail, "Item not found"); - } - - - private TObject? FindByUdi(Udi udi) - { - switch (udi) - { - case GuidUdi guidUdi: - return GetFromService(guidUdi.Guid); - case StringUdi stringUdi: - return GetFromService(stringUdi.Id); - } - - return default; - } - - /// - /// Calculate any dependencies for any given item based on loaded dependency checkers - /// - /// - /// uSync contains no dependency checkers by default - uSync.Complete will load checkers - /// when installed. - /// - public IEnumerable GetDependencies(Guid key, DependencyFlags flags) - { - if (key == Guid.Empty) - { - return GetContainerDependencies(default, flags); - } - else - { - var item = this.GetFromService(key); - if (item == null) - { - var container = this.GetContainer(key); - if (container != null) - { - return GetContainerDependencies(container, flags); - } - return Enumerable.Empty(); - } - - return GetDependencies(item, flags); - } - } - - /// - /// Calculate any dependencies for any given item based on loaded dependency checkers - /// - /// - /// uSync contains no dependency checkers by default - uSync.Complete will load checkers - /// when installed. - /// - public IEnumerable GetDependencies(int id, DependencyFlags flags) - { - // get them from the root. - if (id == -1) return GetContainerDependencies(default, flags); - - var item = this.GetFromService(id); - if (item == null) - { - var container = this.GetContainer(id); - if (container != null) - { - return GetContainerDependencies(container, flags); - } - - return Enumerable.Empty(); - } - return GetDependencies(item, flags); - } - - private bool HasDependencyCheckers() - => dependencyCheckers != null && dependencyCheckers.Count > 0; - - - /// - /// Calculate any dependencies for any given item based on loaded dependency checkers - /// - /// - /// uSync contains no dependency checkers by default - uSync.Complete will load checkers - /// when installed. - /// - protected IEnumerable GetDependencies(TObject item, DependencyFlags flags) - { - if (item == null || !HasDependencyCheckers()) return Enumerable.Empty(); - - var dependencies = new List(); - foreach (var checker in dependencyCheckers) - { - dependencies.AddRange(checker.GetDependencies(item, flags)); - } - return dependencies; - } - - /// - /// Calculate any dependencies for any given item based on loaded dependency checkers - /// - /// - /// uSync contains no dependency checkers by default - uSync.Complete will load checkers - /// when installed. - /// - private IEnumerable GetContainerDependencies(TContainer? parent, DependencyFlags flags) - { - if (!HasDependencyCheckers()) return Enumerable.Empty(); - - var dependencies = new List(); - - var containers = GetFolders(parent); - if (containers != null && containers.Any()) - { - foreach (var container in containers) - { - dependencies.AddRange(GetContainerDependencies(container, flags)); - } - } - - var children = GetChildItems(parent); - if (children != null && children.Any()) - { - foreach (var child in children) - { - var childItem = GetFromService(child); - if (childItem != null) - { - foreach (var checker in dependencyCheckers) - { - dependencies.AddRange(checker.GetDependencies(childItem, flags)); - } - } - } - } - - return dependencies.DistinctBy(x => x.Udi?.ToString() ?? x.Name).OrderByDescending(x => x.Order); - } - - #region Serializer Calls - - /// - /// call the serializer to get an items xml. - /// - protected SyncAttempt SerializeItem(TObject item, SyncSerializerOptions options) - => serializer.Serialize(item, options); - - /// - /// - /// turn the xml into an item (and optionally save it to umbraco). - /// - protected SyncAttempt DeserializeItem(XElement node, SyncSerializerOptions options) - => serializer.Deserialize(node, options); - - /// - /// perform a second pass on an item you are importing. - /// - protected SyncAttempt DeserializeItemSecondPass(TObject item, XElement node, SyncSerializerOptions options) - => serializer.DeserializeSecondPass(item, node, options); - - private SyncChangeInfo IsItemCurrent(XElement node, SyncSerializerOptions options) - { - var change = new SyncChangeInfo(); - change.CurrentNode = SerializeFromNode(node, options); - change.Change = serializer.IsCurrent(node, change.CurrentNode, options); - return change; - } - private XElement? SerializeFromNode(XElement node, SyncSerializerOptions options) - { - var item = serializer.FindItem(node); - if (item != null) - { - var cultures = node.GetCultures(); - if (!string.IsNullOrWhiteSpace(cultures)) - { - // the cultures we serialize should match any in the file. - // this means we then only check the same values at each end. - options.Settings[Core.uSyncConstants.CultureKey] = cultures; - } - - var attempt = this.SerializeItem(item, options); - if (attempt.Success) return attempt.Item; - } - - return null; - } - - private class SyncChangeInfo - { - public ChangeType Change { get; set; } - public XElement? CurrentNode { get; set; } - } - - /// - /// Find an items UDI value based on the values in the uSync XML node - /// - public Udi? FindFromNode(XElement node) - { - var item = serializer.FindItem(node); - if (item != null) - return Udi.Create(this.EntityType, serializer.ItemKey(item)); - - return null; - } - - /// - /// Calculate the current status of an item compared to the XML in a potential import - /// - public ChangeType GetItemStatus(XElement node) - { - var serializerOptions = new SyncSerializerOptions(SerializerFlags.None, this.DefaultConfig.Settings); - return this.IsItemCurrent(node, serializerOptions).Change; - } - - #endregion - - private string GetNameFromFileOrNode(string filename, XElement node) - { - if (string.IsNullOrWhiteSpace(filename) is true) return node.GetAlias(); - return syncFileService.GetSiteRelativePath(filename); - } - - - /// - /// get thekey for any caches we might call (thread based cache value) - /// - /// - protected string GetCacheKeyBase() - => $"keycache_{this.Alias}_{Thread.CurrentThread.ManagedThreadId}"; - - private string PrepCaches() - { - if (this.serializer is ISyncCachedSerializer cachedSerializer) - cachedSerializer.InitializeCache(); - - // make sure the runtime cache is clean. - var key = GetCacheKeyBase(); - - // this also clears the folder cache - as its a starts with call. - runtimeCache.ClearByKey(key); - return key; - } - - private void CleanCaches(string cacheKey) - { - runtimeCache.ClearByKey(cacheKey); - - if (this.serializer is ISyncCachedSerializer cachedSerializer) - cachedSerializer.DisposeCache(); - - } - - #region roots notifications - - /// - /// check roots isn't blocking the save - /// - public virtual void Handle(SavingNotification notification) - { - if (ShouldBlockRootChanges(notification.SavedEntities)) - { - notification.Cancel = true; - notification.Messages.Add(GetCancelMessageForRoots()); - } - } - - /// - /// check roots isn't blocking the move - /// - public virtual void Handle(MovingNotification notification) - { - if (ShouldBlockRootChanges(notification.MoveInfoCollection.Select(x => x.Entity))) - { - notification.Cancel = true; - notification.Messages.Add(GetCancelMessageForRoots()); - } - } - - /// - /// check roots isn't blocking the delete - /// - public virtual void Handle(DeletingNotification notification) - { - if (ShouldBlockRootChanges(notification.DeletedEntities)) - { - notification.Cancel = true; - notification.Messages.Add(GetCancelMessageForRoots()); - } - } - - /// - /// should we block this event based on the existance or root objects. - /// - protected bool ShouldBlockRootChanges(IEnumerable items) - { - if (!ShouldProcessEvent()) return false; - - if (uSyncConfig.Settings.LockRoot == false) return false; - - if (!HasRootFolders()) return false; - - foreach (var item in items) - { - if (RootItemExists(item)) - return true; - } - - return false; - } - - /// - /// get the message we use for cancellations - /// - protected EventMessage GetCancelMessageForRoots() - => new EventMessage("Blocked", "You cannot make this change, root level items are locked", EventMessageType.Error); - - - private bool HasRootFolders() - => syncFileService.AnyFolderExists(uSyncConfig.GetFolders()[..^1]); - - private bool RootItemExists(TObject item) - { - foreach (var folder in uSyncConfig.GetFolders()[..^1]) - { - var filename = GetPath( - Path.Combine(folder, DefaultFolder), - item, - DefaultConfig.GuidNames, - DefaultConfig.UseFlatStructure) - .ToAppSafeFileName(); - - if (syncFileService.FileExists(filename)) - return true; - - } - - return false; - } - - #endregion + /// + /// Reference to the Logger + /// + protected readonly ILogger> logger; + + /// + /// Reference to the uSyncFileService + /// + protected readonly SyncFileService syncFileService; + + /// + /// Reference to the Event service used to handle locking + /// + protected readonly uSyncEventService _mutexService; + + /// + /// List of dependency checkers for this Handler + /// + protected readonly IList> dependencyCheckers; + + /// + /// List of change trackers for this handler + /// + protected readonly IList> trackers; + + /// + /// The Serializer to use for importing/exporting items + /// + protected ISyncSerializer serializer; + + /// + /// Runtime cache for caching lookups + /// + protected readonly IAppPolicyCache runtimeCache; + + /// + /// Alias of the handler, used when getting settings from the configuration file + /// + public string Alias { get; private set; } + + /// + /// Name of handler, displayed to user during reports/imports + /// + public string Name { get; private set; } + + /// + /// name of the folder inside the uSync folder where items are stored + /// + public string DefaultFolder { get; private set; } + + /// + /// priority order items are imported in + /// + /// + /// to import before anything else go below USYNC_RESERVED_LOWER (1000) + /// to import after uSync has done all the other things go past USYNC_RESERVED_UPPER (2000) + /// + public int Priority { get; private set; } + + /// + /// Icon displayed on the screen while the import happens. + /// + public string Icon { get; private set; } + + /// + /// does this handler require two passes at the import (e.g data-types import once, and then again after doc-types) + /// + protected bool IsTwoPass = false; + + /// + /// the object type of the item being processed. + /// + public string ItemType { get; protected set; } = typeof(TObject).Name; + + /// + /// Is the handler enabled + /// + public bool Enabled { get; set; } = true; + + + /// + /// the default configuration for this handler + /// + public HandlerSettings DefaultConfig { get; set; } + + /// + /// the root folder for the handler (based on the settings) + /// + [Obsolete("we should be using the array of folders, will be removed in v15")] + protected string rootFolder { get; set; } + + /// + /// the root folders to use for the handler (based on settings). + /// + protected string[] rootFolders { get; set; } + + /// + /// the UDIEntityType for the handler objects + /// + public string EntityType { get; protected set; } + + /// + /// Name of the type (object) + /// + public string TypeName { get; protected set; } // we calculate these now based on the entityType ? + + /// + /// UmbracoObjectType of items handled by this handler + /// + protected UmbracoObjectTypes itemObjectType { get; set; } = UmbracoObjectTypes.Unknown; + + /// + /// UmbracoObjectType of containers manged by this handler + /// + protected UmbracoObjectTypes itemContainerType = UmbracoObjectTypes.Unknown; + + /// + /// The type of the handler + /// + protected string handlerType; + + /// + /// SyncItem factory reference + /// + protected readonly ISyncItemFactory itemFactory; + + /// + /// Reference to the uSyncConfigService + /// + protected readonly uSyncConfigService uSyncConfig; + + /// + /// Umbraco's shortStringHelper + /// + protected readonly IShortStringHelper shortStringHelper; + + /// + /// Constructor, base for all handlers + /// + public SyncHandlerRoot( + ILogger> logger, + AppCaches appCaches, + IShortStringHelper shortStringHelper, + SyncFileService syncFileService, + uSyncEventService mutexService, + uSyncConfigService uSyncConfig, + ISyncItemFactory itemFactory) + { + this.uSyncConfig = uSyncConfig; + + this.logger = logger; + this.shortStringHelper = shortStringHelper; + this.itemFactory = itemFactory; + + var _serializer = this.itemFactory.GetSerializers().FirstOrDefault(); + if (_serializer is null) + throw new KeyNotFoundException($"No Serializer found for handler {this.Alias}"); + + this.serializer = _serializer; + this.trackers = this.itemFactory.GetTrackers().ToList(); + this.dependencyCheckers = this.itemFactory.GetCheckers().ToList(); + + + this.syncFileService = syncFileService; + this._mutexService = mutexService; + + var currentHandlerType = GetType(); + var meta = currentHandlerType.GetCustomAttribute(false); + if (meta == null) + throw new InvalidOperationException($"The Handler {handlerType} requires a {typeof(SyncHandlerAttribute)}"); + + handlerType = currentHandlerType.ToString(); + Name = meta.Name; + Alias = meta.Alias; + DefaultFolder = meta.Folder; + Priority = meta.Priority; + IsTwoPass = meta.IsTwoPass; + Icon = string.IsNullOrWhiteSpace(meta.Icon) ? "icon-umb-content" : meta.Icon; + EntityType = meta.EntityType; + + TypeName = serializer.ItemType; + + this.itemObjectType = uSyncObjectType.ToUmbracoObjectType(EntityType); + this.itemContainerType = uSyncObjectType.ToContainerUmbracoObjectType(EntityType); + + this.DefaultConfig = GetDefaultConfig(); + rootFolder = uSyncConfig.GetRootFolder(); + rootFolders = uSyncConfig.GetFolders(); + + if (uSyncConfig.Settings.CacheFolderKeys) + { + this.runtimeCache = appCaches.RuntimeCache; + } + else + { + logger.LogInformation("No caching of handler key lookups (CacheFolderKeys = false)"); + this.runtimeCache = NoAppCache.Instance; + } + } + + private HandlerSettings GetDefaultConfig() + { + var defaultSet = uSyncConfig.GetDefaultSetSettings(); + var config = defaultSet.GetHandlerSettings(this.Alias); + + if (defaultSet.DisabledHandlers.InvariantContains(this.Alias)) + config.Enabled = false; + + return config; + } + + #region Importing + + /// + /// Import everything from a given folder, using the supplied configuration settings. + /// + public IEnumerable ImportAll(string folder, HandlerSettings config, bool force, SyncUpdateCallback? callback = null) + => ImportAll([folder], config, new uSyncImportOptions + { + Flags = force ? SerializerFlags.Force : SerializerFlags.None, + Callbacks = new uSyncCallbacks(null, callback) + }); + + /// + /// import everything from a collection of folders, using the supplied config. + /// + /// + /// allows us to 'merge' a collection of folders down and perform an import against them (without first having to actually merge the folders on disk) + /// + public IEnumerable ImportAll(string[] folders, HandlerSettings config, uSyncImportOptions options) + { + var cacheKey = PrepCaches(); + runtimeCache.ClearByKey(cacheKey); + + options.Callbacks?.Update?.Invoke("Calculating import order", 1, 9); + + var items = GetMergedItems(folders); + + options.Callbacks?.Update?.Invoke($"Processing {items.Count} items", 2, 9); + + // create the update list with items.count space. this is the max size we need this list. + List actions = new List(items.Count); + List> updates = new List>(items.Count); + List cleanMarkers = []; + + int count = 0; + int total = items.Count; + + foreach (var item in items) + { + count++; + + options.Callbacks?.Update?.Invoke($"Importing {Path.GetFileNameWithoutExtension(item.Path)}", count, total); + + var result = ImportElement(item.Node, item.Path, config, options); + foreach (var attempt in result) + { + if (attempt.Success) + { + if (attempt.Change == ChangeType.Clean) + { + cleanMarkers.Add(item.Path); + } + else if (attempt.Item is not null && attempt.Item is TObject update) + { + updates.Add(new ImportedItem(item.Node, update)); + } + } + + if (attempt.Change != ChangeType.Clean) + actions.Add(attempt); + } + } + + // clean up memory we didn't use in the update list. + updates.TrimExcess(); + + // bulk save? + if (updates.Count > 0) + { + if (options.Flags.HasFlag(SerializerFlags.DoNotSave)) + { + serializer.Save(updates.Select(x => x.Item)); + } + + PerformSecondPassImports(updates, actions, config, options.Callbacks?.Update); + } + + if (actions.All(x => x.Success) && cleanMarkers.Count > 0) + { + PerformImportClean(cleanMarkers, actions, config, options.Callbacks?.Update); + } + + CleanCaches(cacheKey); + options.Callbacks?.Update?.Invoke("Done", 3, 3); + + return actions; + } + + /// + /// get all items for the report/import process. + /// + /// + /// + public IReadOnlyList FetchAllNodes(string[] folders) + => GetMergedItems(folders); + + /// + /// method to get the merged folders, handlers that care about orders should override this. + /// + protected virtual IReadOnlyList GetMergedItems(string[] folders) + { + var baseTracker = trackers.FirstOrDefault() as ISyncTrackerBase; + return syncFileService.MergeFolders(folders, uSyncConfig.Settings.DefaultExtension, baseTracker).ToArray(); + } + + private void PerformImportClean(List cleanMarkers, List actions, HandlerSettings config, SyncUpdateCallback? callback) + { + foreach (var item in cleanMarkers.Select((filePath, Index) => new { filePath, Index })) + { + var folderName = Path.GetFileName(item.filePath); + callback?.Invoke($"Cleaning {folderName}", item.Index, cleanMarkers.Count); + + var cleanActions = CleanFolder(item.filePath, false, config.UseFlatStructure); + if (cleanActions.Any()) + { + actions.AddRange(cleanActions); + } + else + { + // nothing to delete, we report this as a no change + actions.Add(uSyncAction.SetAction( + success: true, + name: $"Folder {Path.GetFileName(item.filePath)}", + change: ChangeType.NoChange, + filename: syncFileService.GetSiteRelativePath(item.filePath))); + } + } + // remove the actual cleans (they will have been replaced by the deletes + actions.RemoveAll(x => x.Change == ChangeType.Clean); + } + + /// + /// Import everything in a given (child) folder, based on setting + /// + [Obsolete("Import folder method not called directly from v13.1 will be removed in v15")] + protected virtual IEnumerable ImportFolder(string folder, HandlerSettings config, Dictionary updates, bool force, SyncUpdateCallback? callback) + { + List actions = new List(); + var files = GetImportFiles(folder); + + var flags = SerializerFlags.None; + if (force) flags |= SerializerFlags.Force; + + var cleanMarkers = new List(); + + int count = 0; + int total = files.Count(); + foreach (string file in files) + { + count++; + + callback?.Invoke($"Importing {Path.GetFileNameWithoutExtension(file)}", count, total); + + var result = Import(file, config, flags); + foreach (var attempt in result) + { + if (attempt.Success) + { + if (attempt.Change == ChangeType.Clean) + { + cleanMarkers.Add(file); + } + else if (attempt.Item != null && attempt.Item is TObject item) + { + updates.Add(file, item); + } + } + + if (attempt.Change != ChangeType.Clean) + actions.Add(attempt); + } + } + + // bulk save .. + if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any()) + { + // callback?.Invoke($"Saving {updates.Count()} changes", 1, 1); + serializer.Save(updates.Select(x => x.Value)); + } + + var folders = syncFileService.GetDirectories(folder); + foreach (var children in folders) + { + actions.AddRange(ImportFolder(children, config, updates, force, callback)); + } + + if (actions.All(x => x.Success) && cleanMarkers.Count > 0) + { + foreach (var item in cleanMarkers.Select((filePath, Index) => new { filePath, Index })) + { + var folderName = Path.GetFileName(item.filePath); + callback?.Invoke($"Cleaning {folderName}", item.Index, cleanMarkers.Count); + + var cleanActions = CleanFolder(item.filePath, false, config.UseFlatStructure); + if (cleanActions.Any()) + { + actions.AddRange(cleanActions); + } + else + { + // nothing to delete, we report this as a no change + actions.Add(uSyncAction.SetAction( + success: true, + name: $"Folder {Path.GetFileName(item.filePath)}", + change: ChangeType.NoChange, filename: syncFileService.GetSiteRelativePath(item.filePath) + ) + ); + } + } + // remove the actual cleans (they will have been replaced by the deletes + actions.RemoveAll(x => x.Change == ChangeType.Clean); + } + + return actions; + } + + /// + /// Import a single item, from the .config file supplied + /// + public virtual IEnumerable Import(string filePath, HandlerSettings config, SerializerFlags flags) + { + try + { + syncFileService.EnsureFileExists(filePath); + var node = syncFileService.LoadXElement(filePath); + return Import(node, filePath, config, flags); + } + catch (FileNotFoundException notFoundException) + { + return uSyncAction.Fail(Path.GetFileName(filePath), this.handlerType, this.ItemType, ChangeType.Fail, $"File not found {notFoundException.Message}", notFoundException) + .AsEnumerableOfOne(); + } + catch (Exception ex) + { + logger.LogWarning("{alias}: Import Failed : {exception}", this.Alias, ex.ToString()); + return uSyncAction.Fail(Path.GetFileName(filePath), this.handlerType, this.ItemType, ChangeType.Fail, $"Import Fail: {ex.Message}", new Exception(ex.Message, ex)) + .AsEnumerableOfOne(); + } + } + + /// + /// Import a single item based on already loaded XML + /// + public virtual IEnumerable Import(XElement node, string filename, HandlerSettings config, SerializerFlags flags) + { + if (config.FailOnMissingParent) flags |= SerializerFlags.FailMissingParent; + return ImportElement(node, filename, config, new uSyncImportOptions { Flags = flags }); + } + + /// + /// Import a single item from a usync XML file + /// + virtual public IEnumerable Import(string file, HandlerSettings config, bool force) + { + var flags = SerializerFlags.OnePass; + if (force) flags |= SerializerFlags.Force; + + return Import(file, config, flags); + } + + /// + /// Import a node, with settings and options + /// + /// + /// All Imports lead here + /// + virtual public IEnumerable ImportElement(XElement node, string filename, HandlerSettings settings, uSyncImportOptions options) + { + if (!ShouldImport(node, settings)) + { + return uSyncAction.SetAction(true, node.GetAlias(), message: "Change blocked (based on configuration)") + .AsEnumerableOfOne(); + } + + if (_mutexService.FireItemStartingEvent(new uSyncImportingItemNotification(node, (ISyncHandler)this))) + { + // blocked + return uSyncActionHelper + .ReportAction(ChangeType.NoChange, node.GetAlias(), node.GetPath(), GetNameFromFileOrNode(filename, node), node.GetKey(), this.Alias, "Change stopped by delegate event") + .AsEnumerableOfOne(); + } + + try + { + // merge the options from the handler and any import options into our serializer options. + var serializerOptions = new SyncSerializerOptions(options.Flags, settings.Settings, options.UserId); + serializerOptions.MergeSettings(options.Settings); + + // get the item. + var attempt = DeserializeItem(node, serializerOptions); + var action = uSyncActionHelper.SetAction(attempt, GetNameFromFileOrNode(filename, node), node.GetKey(), this.Alias, IsTwoPass); + + // add item if we have it. + if (attempt.Item != null) action.Item = attempt.Item; + + // add details if we have them + if (attempt.Details != null && attempt.Details.Any()) action.Details = attempt.Details; + + // this might not be the place to do this because, two pass items are imported at another point too. + _mutexService.FireItemCompletedEvent(new uSyncImportedItemNotification(node, attempt.Change)); + + + return action.AsEnumerableOfOne(); + } + catch (Exception ex) + { + logger.LogWarning("{alias}: Import Failed : {exception}", this.Alias, ex.ToString()); + return uSyncAction.Fail(Path.GetFileName(filename), this.handlerType, this.ItemType, ChangeType.Fail, + $"{this.Alias} Import Fail: {ex.Message}", new Exception(ex.Message)) + .AsEnumerableOfOne(); + } + + } + + + /// + /// Works through a list of items that have been processed and performs the second import pass on them. + /// + private void PerformSecondPassImports(List> importedItems, List actions, HandlerSettings config, SyncUpdateCallback? callback = null) + { + foreach (var item in importedItems.Select((update, Index) => new { update, Index })) + { + var itemKey = item.update.Node.GetKey(); + + callback?.Invoke($"Second Pass {item.update.Node.GetKey()}", item.Index, importedItems.Count); + var attempt = ImportSecondPass(item.update.Node, item.update.Item, config, callback); + if (attempt.Success) + { + // if the second attempt has a message on it, add it to the first attempt. + if (!string.IsNullOrWhiteSpace(attempt.Message) || attempt.Details?.Any() == true) + { + // uSyncAction action = actions.FirstOrDefault(x => $"{x.key}_{x.HandlerAlias}" == $"{itemKey}_{this.Alias}", new uSyncAction { key = Guid.Empty }); + if (actions.TryFindAction(itemKey, this.Alias, out var action)) + { + if (action.key != Guid.Empty) + { + actions.Remove(action); + action.Message += attempt.Message ?? ""; + + if (attempt.Details?.Any() == true) + { + var details = action.Details?.ToList() ?? []; + details.AddRange(attempt.Details); + action.Details = details; + } + actions.Add(action); + } + } + } + if (attempt.Change > ChangeType.NoChange && !attempt.Saved && attempt.Item != null) + { + serializer.Save(attempt.Item.AsEnumerableOfOne()); + } + } + else + { + if (actions.TryFindAction(itemKey, this.Alias, out var action)) + { + actions.Remove(action); + action.Success = attempt.Success; + action.Message = $"Second Pass Fail: {attempt.Message}"; + action.Exception = attempt.Exception; + actions.Add(action); + } + } + } + } + + /// + /// Perform a second pass import on an item + /// + virtual public IEnumerable ImportSecondPass(uSyncAction action, HandlerSettings settings, uSyncImportOptions options) + { + if (!IsTwoPass) return Enumerable.Empty(); + + try + { + var fileName = action.FileName; + + if (fileName is null || syncFileService.FileExists(fileName) is false) + return Enumerable.Empty(); + + var node = syncFileService.LoadXElement(fileName); + var item = GetFromService(node.GetKey()); + if (item == null) return Enumerable.Empty(); + + // merge the options from the handler and any import options into our serializer options. + var serializerOptions = new SyncSerializerOptions(options?.Flags ?? SerializerFlags.None, settings.Settings, options?.UserId ?? -1); + serializerOptions.MergeSettings(options?.Settings); + + // do the second pass on this item + var result = DeserializeItemSecondPass(item, node, serializerOptions); + + return uSyncActionHelper.SetAction(result, syncFileService.GetSiteRelativePath(fileName), node.GetKey(), this.Alias).AsEnumerableOfOne(); + } + catch (Exception ex) + { + logger.LogWarning($"Second Import Failed: {ex}"); + return uSyncAction.Fail(action.Name, this.handlerType, action.ItemType, ChangeType.ImportFail, "Second import failed", ex).AsEnumerableOfOne(); + } + } + + + /// + /// Perform a 'second pass' import on a single item. + /// + [Obsolete("Call method with node element to reduce disk IO, will be removed in v15")] + virtual public SyncAttempt ImportSecondPass(string file, TObject item, HandlerSettings config, SyncUpdateCallback callback) + { + if (IsTwoPass) + { + try + { + syncFileService.EnsureFileExists(file); + + var flags = SerializerFlags.None; + + var node = syncFileService.LoadXElement(file); + return DeserializeItemSecondPass(item, node, new SyncSerializerOptions(flags, config.Settings)); + } + catch (Exception ex) + { + logger.LogWarning($"Second Import Failed: {ex.ToString()}"); + return SyncAttempt.Fail(GetItemAlias(item), item, ChangeType.Fail, ex.Message, ex); + } + } + + return SyncAttempt.Succeed(GetItemAlias(item), ChangeType.NoChange); + } + + /// + /// Perform a 'second pass' import on a single item. + /// + virtual public SyncAttempt ImportSecondPass(XElement node, TObject item, HandlerSettings config, SyncUpdateCallback? callback) + { + if (IsTwoPass is false) + return SyncAttempt.Succeed(GetItemAlias(item), ChangeType.NoChange); + + try + { + return DeserializeItemSecondPass(item, node, new SyncSerializerOptions(SerializerFlags.None, config.Settings)); + } + catch (Exception ex) + { + logger.LogWarning($"Second Import Failed: {ex.ToString()}"); + return SyncAttempt.Fail(GetItemAlias(item), item, ChangeType.Fail, ex.Message, ex); + } + } + + /// + /// given a folder we calculate what items we can remove, because they are + /// not in one the files in the folder. + /// + protected virtual IEnumerable CleanFolder(string cleanFile, bool reportOnly, bool flat) + { + var folder = Path.GetDirectoryName(cleanFile); + if (string.IsNullOrWhiteSpace(folder) is true || syncFileService.DirectoryExists(folder) is false) + return Enumerable.Empty(); + + // get the keys for every item in this folder. + + // this would works on the flat folder structure too, + // there we are being super defensive, so if an item + // is anywhere in the folder it won't get removed + // even if the folder is wrong + // be a little slower (not much though) + + // we cache this, (it is cleared on an ImportAll) + var keys = GetFolderKeys(folder, flat); + if (keys.Count > 0) + { + // move parent to here, we only need to check it if there are files. + var parent = GetCleanParent(cleanFile); + if (parent == null) return Enumerable.Empty(); + + logger.LogDebug("Got parent with {alias} from clean file {file}", GetItemAlias(parent), Path.GetFileName(cleanFile)); + + // keys should aways have at least one entry (the key from cleanFile) + // if it doesn't then something might have gone wrong. + // because we are being defensive when it comes to deletes, + // we only then do deletes when we know we have loaded some keys! + return DeleteMissingItems(parent, keys, reportOnly); + } + else + { + logger.LogWarning("Failed to get the keys for items in the folder, there might be a disk issue {folder}", folder); + return Enumerable.Empty(); + } + } + + /// + /// pre-populates the cache folder key list. + /// + /// + /// this means if we are calling the process multiple times, + /// we can optimise the key code and only load it once. + /// + public void PreCacheFolderKeys(string folder, IList folderKeys) + { + var cacheKey = $"{GetCacheKeyBase()}_{folder.GetHashCode()}"; + runtimeCache.ClearByKey(cacheKey); + runtimeCache.GetCacheItem(cacheKey, () => folderKeys); + } + + /// + /// Get the GUIDs for all items in a folder + /// + /// + /// This is disk intensive, (checking the .config files all the time) + /// so we cache it, and if we are using the flat folder structure, then + /// we only do it once, so its quicker. + /// + protected IList GetFolderKeys(string folder, bool flat) + { + // We only need to load all the keys once per handler (if all items are in a folder that key will be used). + var folderKey = folder.GetHashCode(); + + var cacheKey = $"{GetCacheKeyBase()}_{folderKey}"; + + + return runtimeCache.GetCacheItem(cacheKey, () => + { + logger.LogDebug("Getting Folder Keys : {cacheKey}", cacheKey); + + // when it's not flat structure we also get the sub folders. (extra defensive get them all) + var keys = new List(); + var files = syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}", !flat).ToList(); + + foreach (var file in files) + { + var node = XElement.Load(file); + var key = node.GetKey(); + if (key != Guid.Empty && !keys.Contains(key)) + { + keys.Add(key); + } + } + + logger.LogDebug("Loaded {count} keys from {folder} [{cacheKey}]", keys.Count, folder, cacheKey); + + return keys; + + }, null) ?? []; + } + + /// + /// Get the parent item of the clean file (so we can check if the folder has any versions of this item in it) + /// + protected TObject? GetCleanParent(string file) + { + var node = XElement.Load(file); + var key = node.GetKey(); + if (key == Guid.Empty) return default; + return GetFromService(key); + } + + /// + /// remove an items that are not listed in the GUIDs to keep + /// + /// parent item that all keys will be under + /// list of GUIDs of items we don't want to delete + /// will just report what would happen (doesn't do the delete) + /// list of delete actions + protected abstract IEnumerable DeleteMissingItems(TObject parent, IEnumerable keysToKeep, bool reportOnly); + + /// + /// Remove an items that are not listed in the GUIDs to keep. + /// + /// parent item that all keys will be under + /// list of GUIDs of items we don't want to delete + /// will just report what would happen (doesn't do the delete) + /// list of delete actions + protected virtual IEnumerable DeleteMissingItems(int parentId, IEnumerable keysToKeep, bool reportOnly) + => Enumerable.Empty(); + + /// + /// Get the files we are going to import from a folder. + /// + protected virtual IEnumerable GetImportFiles(string folder) + => syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}").OrderBy(x => x); + + /// + /// check to see if this element should be imported as part of the process. + /// + virtual protected bool ShouldImport(XElement node, HandlerSettings config) + { + // if createOnly is on, then we only create things that are not already there. + // this lookup is slow (relatively) so we only do it if we have to. + if (config.GetSetting(Core.uSyncConstants.DefaultSettings.CreateOnly, Core.uSyncConstants.DefaultSettings.CreateOnly_Default) + || config.GetSetting(Core.uSyncConstants.DefaultSettings.OneWay, Core.uSyncConstants.DefaultSettings.CreateOnly_Default)) + { + var item = serializer.FindItem(node); + if (item != null) + { + logger.LogDebug("CreateOnly: Item {alias} already exist not importing it.", node.GetAlias()); + return false; + } + } + + + // Ignore alias setting. + // if its set the thing with this alias is ignored. + var ignore = config.GetSetting("IgnoreAliases", string.Empty); + if (!string.IsNullOrWhiteSpace(ignore)) + { + var ignoreList = ignore.ToDelimitedList(); + if (ignoreList.InvariantContains(node.GetAlias())) + { + logger.LogDebug("Ignore: Item {alias} is in the ignore list", node.GetAlias()); + return false; + } + } + + + return true; + } + + + /// + /// Check to see if this element should be exported. + /// + virtual protected bool ShouldExport(XElement node, HandlerSettings config) => true; + + #endregion + + #region Exporting + + /// + /// Export all items to a give folder on the disk + /// + virtual public IEnumerable ExportAll(string folder, HandlerSettings config, SyncUpdateCallback? callback) + => ExportAll([folder], config, callback); + + /// + /// Export all items to a give folder on the disk + /// + virtual public IEnumerable ExportAll(string[] folders, HandlerSettings config, SyncUpdateCallback? callback) + { + // we don't clean the folder out on an export all. + // because the actions (renames/deletes) live in the folder + // + // there will have to be a different clean option + // syncFileService.CleanFolder(folder); + + return ExportAll(default, folders, config, callback); + } + + /// + /// export all items underneath a given container + /// + virtual public IEnumerable ExportAll(TContainer? parent, string folder, HandlerSettings config, SyncUpdateCallback? callback) + => ExportAll(parent, [folder], config, callback); + + /// + /// Export all items to a give folder on the disk + /// + virtual public IEnumerable ExportAll(TContainer? parent, string[] folders, HandlerSettings config, SyncUpdateCallback? callback) + { + var actions = new List(); + + if (itemContainerType != UmbracoObjectTypes.Unknown) + { + var containers = GetFolders(parent); + foreach (var container in containers) + { + actions.AddRange(ExportAll(container, folders, config, callback)); + } + } + + var items = GetChildItems(parent).ToList(); + foreach (var item in items.Select((Value, Index) => new { Value, Index })) + { + TObject? concreteType; + if (item.Value is TObject t) + { + concreteType = t; + } + else + { + concreteType = GetFromService(item.Value); + } + if (concreteType is not null) + { // only export the items (not the containers). + callback?.Invoke(GetItemName(concreteType), item.Index, items.Count); + actions.AddRange(Export(concreteType, folders, config)); + } + actions.AddRange(ExportAll(item.Value, folders, config, callback)); + } + + return actions; + } + + /// + /// Fetch all child items beneath a given container + /// + abstract protected IEnumerable GetChildItems(TContainer? parent); + + /// + /// Fetch all child items beneath a given folder + /// + /// + /// + abstract protected IEnumerable GetFolders(TContainer? parent); + + /// + /// Does this container have any children + /// + public bool HasChildren(TContainer item) + => GetFolders(item).Any() || GetChildItems(item).Any(); + + + /// + /// Export a single item based on it's ID + /// + public IEnumerable Export(int id, string folder, HandlerSettings settings) + => Export(id, [folder], settings); + + /// + /// Export an item based on its id, observing root behavior. + /// + public IEnumerable Export(int id, string[] folders, HandlerSettings settings) + { + var item = this.GetFromService(id); + if (item is null) + { + return uSyncAction.Fail( + id.ToString(), this.handlerType, this.ItemType, + ChangeType.Export, "Unable to find item", + new KeyNotFoundException($"Item of {id} cannot be found")) + .AsEnumerableOfOne(); + } + return this.Export(item, folders, settings); + } + + /// + /// Export an single item from a given UDI value + /// + public IEnumerable Export(Udi udi, string folder, HandlerSettings settings) + => Export(udi, [folder], settings); + + /// + /// Export an single item from a given UDI value + /// + public IEnumerable Export(Udi udi, string[] folders, HandlerSettings settings) + { + var item = FindByUdi(udi); + if (item != null) + return Export(item, folders, settings); + + return uSyncAction.Fail(nameof(udi), this.handlerType, this.ItemType, ChangeType.Fail, $"Item not found {udi}", + new KeyNotFoundException(nameof(udi))) + .AsEnumerableOfOne(); + } + + /// + /// Export a given item to disk + /// + virtual public IEnumerable Export(TObject item, string folder, HandlerSettings config) + => Export(item, [folder], config); + + /// + /// Export a given item to disk + /// + virtual public IEnumerable Export(TObject item, string[] folders, HandlerSettings config) + { + if (item == null) + return uSyncAction.Fail(nameof(item), this.handlerType, this.ItemType, ChangeType.Fail, "Item not set", + new ArgumentNullException(nameof(item))).AsEnumerableOfOne(); + + if (_mutexService.FireItemStartingEvent(new uSyncExportingItemNotification(item, (ISyncHandler)this))) + { + return uSyncActionHelper + .ReportAction(ChangeType.NoChange, GetItemName(item), string.Empty, string.Empty, GetItemKey(item), this.Alias, + "Change stopped by delegate event") + .AsEnumerableOfOne(); + } + + var targetFolder = folders.Last(); + + var filename = GetPath(targetFolder, item, config.GuidNames, config.UseFlatStructure) + .ToAppSafeFileName(); + + // + if (IsLockedAtRoot(folders, filename.Substring(targetFolder.Length + 1))) + { + // if we have lock roots on, then this item will not export + // because exporting would mean the root was no longer used. + return uSyncAction.SetAction(true, syncFileService.GetSiteRelativePath(filename), + type: typeof(TObject).ToString(), + change: ChangeType.NoChange, + message: "Not exported (would overwrite root value)", + filename: filename).AsEnumerableOfOne(); + } + + + var attempt = Export_DoExport(item, filename, folders, config); + + if (attempt.Change > ChangeType.NoChange) + _mutexService.FireItemCompletedEvent(new uSyncExportedItemNotification(attempt.Item, ChangeType.Export)); + + return uSyncActionHelper.SetAction(attempt, syncFileService.GetSiteRelativePath(filename), GetItemKey(item), this.Alias).AsEnumerableOfOne(); + } + + /// + /// Do the meat of the export + /// + /// + /// inheriting this method, means you don't have to repeat all the checks in child handlers. + /// + protected virtual SyncAttempt Export_DoExport(TObject item, string filename, string[] folders, HandlerSettings config) + { + var attempt = SerializeItem(item, new SyncSerializerOptions(config.Settings)); + if (attempt.Success && attempt.Item is not null) + { + if (ShouldExport(attempt.Item, config)) + { + // only write the file to disk if it should be exported. + syncFileService.SaveXElement(attempt.Item, filename); + + if (config.CreateClean && HasChildren(item)) + { + CreateCleanFile(GetItemKey(item), filename); + } + } + else + { + return SyncAttempt.Succeed(Path.GetFileName(filename), ChangeType.NoChange, "Not Exported (Based on configuration)"); + } + } + + return attempt; + } + + /// + /// Checks to see if this item is locked at the root level (meaning we shouldn't save it) + /// + protected bool IsLockedAtRoot(string[] folders, string path) + { + if (folders.Length <= 1) return false; + + if (ExistsInFolders(folders[..^1], path.TrimStart(['\\', '/']))) + { + return uSyncConfig.Settings.LockRoot || uSyncConfig.Settings.LockRootTypes.InvariantContains(EntityType); + } + + return false; + + bool ExistsInFolders(string[] folders, string path) + { + foreach (var folder in folders) + { + if (syncFileService.FileExists(Path.Combine(folder, path.TrimStart(['\\', '/'])))) + { + return true; + } + } + + return false; + } + } + + /// + /// does this item have any children ? + /// + /// + /// on items where we can check this (quickly) we can reduce the number of checks we might + /// make on child items or cleaning up where we don't need to. + /// + protected virtual bool HasChildren(TObject item) + => true; + + /// + /// create a clean file, which is used as a marker, when performing remote deletes. + /// + protected void CreateCleanFile(Guid key, string filename) + { + if (string.IsNullOrWhiteSpace(filename) || key == Guid.Empty) + return; + + var folder = Path.GetDirectoryName(filename); + var name = Path.GetFileNameWithoutExtension(filename); + + if (string.IsNullOrEmpty(folder)) return; + + var cleanPath = Path.Combine(folder, $"{name}_clean.config"); + + var node = XElementExtensions.MakeEmpty(key, SyncActionType.Clean, $"clean {name} children"); + node.Add(new XAttribute("itemType", serializer.ItemType)); + syncFileService.SaveXElement(node, cleanPath); + } + + #endregion + + #region Reporting + + /// + /// Run a report based on a given folder + /// + public IEnumerable Report(string folder, HandlerSettings config, SyncUpdateCallback? callback) + => Report([folder], config, callback); + + /// + /// Run a report based on a set of folders. + /// + public IEnumerable Report(string[] folders, HandlerSettings config, SyncUpdateCallback? callback) + { + List actions = []; + + var cacheKey = PrepCaches(); + + callback?.Invoke("Organising import structure", 1, 3); + + var items = GetMergedItems(folders); + + int count = 0; + + foreach (var item in items) + { + count++; + callback?.Invoke(Path.GetFileNameWithoutExtension(item.Path), count, items.Count); + actions.AddRange(ReportElement(item.Node, item.FileName, config)); + } + + callback?.Invoke("Validating Report", 2, 3); + var validationActions = ReportMissingParents(actions.ToArray()); + actions.AddRange(ReportDeleteCheck(uSyncConfig.GetRootFolder(), validationActions)); + + CleanCaches(cacheKey); + callback?.Invoke("Done", 3, 3); + return actions; + } + + private List ValidateReport(string folder, List actions) + { + // Alters the existing list, by changing the type as needed. + var validationActions = ReportMissingParents(actions.ToArray()); + + // adds new actions - for delete clashes. + validationActions.AddRange(ReportDeleteCheck(folder, validationActions)); + + return validationActions; + } + + /// + /// Check to returned report to see if there is a delete and an update for the same item + /// because if there is then we have issues. + /// + protected virtual IEnumerable ReportDeleteCheck(string folder, IEnumerable actions) + { + var duplicates = new List(); + + // delete checks. + foreach (var deleteAction in actions.Where(x => x.Change != ChangeType.NoChange && x.Change == ChangeType.Delete)) + { + // todo: this is only matching by key, but non-tree based serializers also delete by alias. + // so this check actually has to be booted back down to the serializer. + if (actions.Any(x => x.Change != ChangeType.Delete && DoActionsMatch(x, deleteAction))) + { + var duplicateAction = uSyncActionHelper.ReportActionFail(deleteAction.Name, + $"Duplicate! {deleteAction.Name} exists both as delete and import action"); + + // create a detail message to tell people what has happened. + duplicateAction.DetailMessage = "uSync detected a duplicate actions, where am item will be both created and deleted."; + var details = new List(); + + // add the delete message to the list of changes + var filename = Path.GetFileName(deleteAction.FileName) ?? string.Empty; + var relativePath = deleteAction.FileName?.Substring(folder.Length) ?? string.Empty; + + details.Add(uSyncChange.Delete(filename, $"Delete: {deleteAction.Name} ({filename}", relativePath)); + + // add all the duplicates to the list of changes. + foreach (var dup in actions.Where(x => x.Change != ChangeType.Delete && DoActionsMatch(x, deleteAction))) + { + var dupFilename = Path.GetFileName(dup.FileName) ?? string.Empty; + var dupRelativePath = dup.FileName?.Substring(folder.Length) ?? string.Empty; + + details.Add( + uSyncChange.Update( + path: dupFilename, + name: $"{dup.Change} : {dup.Name} ({dupFilename})", + oldValue: "", + newValue: dupRelativePath)); + } + + duplicateAction.Details = details; + duplicates.Add(duplicateAction); + } + } + + return duplicates; + } + + /// + /// check to see if an action matches, another action. + /// + /// + /// how two actions match can vary based on handler, in the most part they are matched by key + /// but some items will also check based on the name. + /// + /// when we are dealing with handlers where things can have the same + /// name (tree items, such as content or media), this function has + /// to be overridden to remove the name check. + /// + protected virtual bool DoActionsMatch(uSyncAction a, uSyncAction b) + { + if (a.key == b.key) return true; + if (a.Name.Equals(b.Name, StringComparison.InvariantCultureIgnoreCase)) return true; + return false; + } + + /// + /// check if a node matches a item + /// + /// + /// Like above we want to match on key and alias, but only if the alias is unique. + /// however the GetItemAlias function is overridden by tree based handlers to return a unique + /// alias (the key again), so we don't get false positives. + /// + protected virtual bool DoItemsMatch(XElement node, TObject item) + { + if (GetItemKey(item) == node.GetKey()) return true; + + // yes this is an or, we've done it explicitly, so you can tell! + if (node.GetAlias().Equals(this.GetItemAlias(item), StringComparison.InvariantCultureIgnoreCase)) return true; + + return false; + } + + /// + /// Check report for any items that are missing their parent items + /// + /// + /// The serializers will report if an item is missing a parent item within umbraco, + /// but because the serializer isn't aware of the wider import (all the other items) + /// it can't say if the parent is in the import. + /// + /// This method checks for the parent of an item in the wider list of items being + /// imported. + /// + private List ReportMissingParents(uSyncAction[] actions) + { + for (int i = 0; i < actions.Length; i++) + { + if (actions[i].Change != ChangeType.ParentMissing || actions[i].FileName is null) continue; + + var node = XElement.Load(actions[i].FileName!); + var guid = node.GetParentKey(); + + if (guid != Guid.Empty) + { + if (actions.Any(x => x.key == guid && (x.Change < ChangeType.Fail || x.Change == ChangeType.ParentMissing))) + { + logger.LogDebug("Found existing key in actions {item}", actions[i].Name); + actions[i].Change = ChangeType.Create; + } + else + { + logger.LogWarning("{item} is missing a parent", actions[i].Name); + } + } + } + + return actions.ToList(); + } + + /// + /// Run a report on a given folder + /// + public virtual IEnumerable ReportFolder(string folder, HandlerSettings config, SyncUpdateCallback? callback) + { + + List actions = new List(); + + var files = GetImportFiles(folder).ToList(); + + int count = 0; + + logger.LogDebug("ReportFolder: {folder} ({count} files)", folder, files.Count); + + foreach (string file in files) + { + count++; + callback?.Invoke(Path.GetFileNameWithoutExtension(file), count, files.Count); + + actions.AddRange(ReportItem(file, config)); + } + + foreach (var children in syncFileService.GetDirectories(folder)) + { + actions.AddRange(ReportFolder(children, config, callback)); + } + + return actions; + } + + /// + /// Report on any changes for a single XML node. + /// + protected virtual IEnumerable ReportElement(XElement node, string filename, HandlerSettings? config) + => ReportElement(node, filename, config ?? this.DefaultConfig, new uSyncImportOptions()); + + + /// + /// Report an Element + /// + public IEnumerable ReportElement(XElement node, string filename, HandlerSettings settings, uSyncImportOptions options) + { + try + { + // starting reporting notification + // this lets us intercept a report and + // shortcut the checking (sometimes). + if (_mutexService.FireItemStartingEvent(new uSyncReportingItemNotification(node))) + { + return uSyncActionHelper + .ReportAction(ChangeType.NoChange, node.GetAlias(), node.GetPath(), GetNameFromFileOrNode(filename, node), node.GetKey(), this.Alias, + "Change stopped by delegate event") + .AsEnumerableOfOne(); + } + + var actions = new List(); + + // get the serializer options + var serializerOptions = new SyncSerializerOptions(options.Flags, settings.Settings, options.UserId); + serializerOptions.MergeSettings(options.Settings); + + // check if this item is current (the provided XML and exported XML match) + var change = IsItemCurrent(node, serializerOptions); + + var action = uSyncActionHelper + .ReportAction(change.Change, node.GetAlias(), node.GetPath(), GetNameFromFileOrNode(filename, node), node.GetKey(), this.Alias, ""); + + + + action.Message = ""; + + if (action.Change == ChangeType.Clean) + { + actions.AddRange(CleanFolder(filename, true, settings.UseFlatStructure)); + } + else if (action.Change > ChangeType.NoChange) + { + if (change.CurrentNode is not null) + { + action.Details = GetChanges(node, change.CurrentNode, serializerOptions); + if (action.Change != ChangeType.Create && (action.Details == null || action.Details.Count() == 0)) + { + action.Message = "XML is different - but properties may not have changed"; + action.Details = MakeRawChange(node, change.CurrentNode, serializerOptions).AsEnumerableOfOne(); + } + else + { + action.Message = $"{action.Change}"; + } + } + actions.Add(action); + } + else + { + actions.Add(action); + } + + // tell other things we have reported this item. + _mutexService.FireItemCompletedEvent(new uSyncReportedItemNotification(node, action.Change)); + + return actions; + } + catch (FormatException fex) + { + return uSyncActionHelper + .ReportActionFail(Path.GetFileName(node.GetAlias()), $"format error {fex.Message}") + .AsEnumerableOfOne(); + } + } + + private uSyncChange MakeRawChange(XElement node, XElement current, SyncSerializerOptions options) + { + if (current != null) + return uSyncChange.Update(node.GetAlias(), "Raw XML", current.ToString(), node.ToString()); + + return uSyncChange.NoChange(node.GetAlias(), node.GetAlias()); + } + + /// + /// Run a report on a single file. + /// + protected IEnumerable ReportItem(string file, HandlerSettings config) + { + try + { + var node = syncFileService.LoadXElement(file); + + if (ShouldImport(node, config)) + { + return ReportElement(node, file, config); + } + else + { + return uSyncActionHelper.ReportAction(ChangeType.NoChange, node.GetAlias(), node.GetPath(), syncFileService.GetSiteRelativePath(file), node.GetKey(), + this.Alias, "Will not be imported (Based on configuration)") + .AsEnumerableOfOne(); + } + } + catch (Exception ex) + { + return uSyncActionHelper + .ReportActionFail(Path.GetFileName(file), $"Reporting error {ex.Message}") + .AsEnumerableOfOne(); + } + + } + + + private IEnumerable GetChanges(XElement node, XElement currentNode, SyncSerializerOptions options) + => itemFactory.GetChanges(node, currentNode, options); + + #endregion + + #region Notification Events + + /// + /// calculate if this handler should process the events. + /// + /// + /// will check if uSync is paused, the handler is enabled or the action is set. + /// + protected bool ShouldProcessEvent() + { + if (_mutexService.IsPaused) return false; + if (!DefaultConfig.Enabled) return false; + + + var group = !string.IsNullOrWhiteSpace(DefaultConfig.Group) ? DefaultConfig.Group : this.Group; + + if (uSyncConfig.Settings.ExportOnSave.InvariantContains("All") || + uSyncConfig.Settings.ExportOnSave.InvariantContains(group)) + { + return HandlerActions.Save.IsValidAction(DefaultConfig.Actions); + } + + return false; + } + + /// + /// Handle an Umbraco Delete notification + /// + public virtual void Handle(DeletedNotification notification) + { + if (!ShouldProcessEvent()) return; + + foreach (var item in notification.DeletedEntities) + { + try + { + var handlerFolders = GetDefaultHandlerFolders(); + ExportDeletedItem(item, handlerFolders, DefaultConfig); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to create delete marker"); + notification.Messages.Add(new EventMessage("uSync", $"Failed to mark as deleted : {ex.Message}", EventMessageType.Warning)); + } + } + } + + /// + /// Handle the Umbraco Saved notification for items. + /// + /// + public virtual void Handle(SavedNotification notification) + { + if (!ShouldProcessEvent()) return; + if (notification.State.TryGetValue(uSync.EventPausedKey, out var paused) && paused is true) + return; + + var handlerFolders = GetDefaultHandlerFolders(); + + foreach (var item in notification.SavedEntities) + { + try + { + var attempts = Export(item, handlerFolders, DefaultConfig); + foreach (var attempt in attempts.Where(x => x.Success)) + { + if (attempt.FileName is null) continue; + this.CleanUp(item, attempt.FileName, handlerFolders.Last()); + } + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to create uSync export file"); + notification.Messages.Add(new EventMessage("uSync", $"Failed to create export file : {ex.Message}", EventMessageType.Warning)); + } + } + } + + /// + /// Handle the Umbraco moved notification for items. + /// + /// + public virtual void Handle(MovedNotification notification) + { + try + { + if (!ShouldProcessEvent()) return; + HandleMove(notification.MoveInfoCollection); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to export move operation"); + notification.Messages.Add(new EventMessage("uSync", $"Failed to export move : {ex.Message}", EventMessageType.Warning)); + } + + } + + /// + /// Process a collection of move events + /// + /// + /// This has been separated out, because we also call this code when a handler supports + /// recycle bin events + /// + protected void HandleMove(IEnumerable> moveInfoCollection) + { + foreach (var item in moveInfoCollection) + { + var handlerFolders = GetDefaultHandlerFolders(); + var attempts = Export(item.Entity, handlerFolders, DefaultConfig); + + if (!this.DefaultConfig.UseFlatStructure) + { + // moves only need cleaning up if we are not using flat, because + // with flat the file will always be in the same folder. + + foreach (var attempt in attempts.Where(x => x.Success is true)) + { + if (attempt.FileName is null) continue; + this.CleanUp(item.Entity, attempt.FileName, handlerFolders.Last()); + } + } + } + } + + /// + /// Export any deletes items to disk + /// + /// + /// Deleted items get 'empty' files on disk so we know they where deleted + /// + protected virtual void ExportDeletedItem(TObject item, string folder, HandlerSettings config) + => ExportDeletedItem(item, [folder], config); + + /// + /// Export any deletes items to disk + /// + /// + /// Deleted items get 'empty' files on disk so we know they where deleted + /// + protected virtual void ExportDeletedItem(TObject item, string[] folders, HandlerSettings config) + { + if (item == null) return; + + var targetFolder = folders.Last(); + + var filename = GetPath(targetFolder, item, config.GuidNames, config.UseFlatStructure) + .ToAppSafeFileName(); + + if (IsLockedAtRoot(folders, filename.Substring(targetFolder.Length + 1))) + { + // don't do anything this thing exists at a higher level. ! + return; + } + + + var attempt = serializer.SerializeEmpty(item, SyncActionType.Delete, string.Empty); + if (attempt.Item is not null && ShouldExport(attempt.Item, config) is true) + { + if (attempt.Success && attempt.Change != ChangeType.NoChange) + { + syncFileService.SaveXElement(attempt.Item, filename); + + // so check - it shouldn't (under normal operation) + // be possible for a clash to exist at delete, because nothing else + // will have changed (like name or location) + + // we only then do this if we are not using flat structure. + if (!DefaultConfig.UseFlatStructure) + this.CleanUp(item, filename, Path.Combine(folders.Last(), this.DefaultFolder)); + } + } + } + + /// + /// get all the possible folders for this handlers + /// + protected string[] GetDefaultHandlerFolders() + => rootFolders.Select(f => Path.Combine(f, DefaultFolder)).ToArray(); + + + /// + /// Cleans up the handler folder, removing duplicate files for this item + /// + /// + /// e.g if someone renames a thing (and we are using the name in the file) + /// this will clean anything else in the folder that has that key / alias + /// + protected virtual void CleanUp(TObject item, string newFile, string folder) + { + var physicalFile = syncFileService.GetAbsPath(newFile); + + var files = syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}"); + + foreach (string file in files) + { + // compare the file paths. + if (!syncFileService.PathMatches(physicalFile, file)) // This is not the same file, as we are saving. + { + try + { + var node = syncFileService.LoadXElement(file); + + // if this XML file matches the item we have just saved. + + if (!node.IsEmptyItem() || node.GetEmptyAction() != SyncActionType.Rename) + { + // the node isn't empty, or its not a rename (because all clashes become renames) + + if (DoItemsMatch(node, item)) + { + logger.LogDebug("Duplicate {file} of {alias}, saving as rename", Path.GetFileName(file), this.GetItemAlias(item)); + + var attempt = serializer.SerializeEmpty(item, SyncActionType.Rename, node.GetAlias()); + if (attempt.Success && attempt.Item is not null) + { + syncFileService.SaveXElement(attempt.Item, file); + } + } + } + } + catch (Exception ex) + { + logger.LogWarning("Error during cleanup of existing files {message}", ex.Message); + // cleanup should fail silently ? - because it can impact on normal Umbraco operations? + } + } + } + + var folders = syncFileService.GetDirectories(folder); + foreach (var children in folders) + { + CleanUp(item, newFile, children); + } + } + + #endregion + + // 98% of the time the serializer can do all these calls for us, + // but for blueprints, we want to get different items, (but still use the + // content serializer) so we override them. + + + /// + /// Fetch an item via the Serializer + /// + protected virtual TObject? GetFromService(int id) => serializer.FindItem(id); + + /// + /// Fetch an item via the Serializer + /// + protected virtual TObject? GetFromService(Guid key) => serializer.FindItem(key); + + /// + /// Fetch an item via the Serializer + /// + protected virtual TObject? GetFromService(string alias) => serializer.FindItem(alias); + + /// + /// Delete an item via the Serializer + /// + protected virtual void DeleteViaService(TObject item) => serializer.DeleteItem(item); + + /// + /// Get the alias of an item from the Serializer + /// + protected string GetItemAlias(TObject item) => serializer.ItemAlias(item); + + /// + /// Get the Key of an item from the Serializer + /// + protected Guid GetItemKey(TObject item) => serializer.ItemKey(item); + + /// + /// Get a container item from the Umbraco service. + /// + abstract protected TObject? GetFromService(TContainer? item); + + /// + /// Get a container item from the Umbraco service. + /// + virtual protected TContainer? GetContainer(Guid key) => default; + + /// + /// Get a container item from the Umbraco service. + /// + virtual protected TContainer? GetContainer(int id) => default; + + /// + /// Get the file path to use for an item + /// + /// Item to derive path for + /// should we use the key value in the path + /// should the file be flat and ignore any sub folders? + /// relative file path to use for an item + virtual protected string GetItemPath(TObject item, bool useGuid, bool isFlat) + => useGuid ? GetItemKey(item).ToString() : GetItemFileName(item); + + /// + /// Get the file name to use for an item + /// + virtual protected string GetItemFileName(TObject item) + => GetItemAlias(item).ToSafeFileName(shortStringHelper); + + /// + /// Get the name of a supplied item + /// + abstract protected string GetItemName(TObject item); + + /// + /// Calculate the relative Physical path value for any item + /// + /// + /// this is where a item is saved on disk in relation to the uSync folder + /// + virtual protected string GetPath(string folder, TObject item, bool GuidNames, bool isFlat) + { + if (isFlat && GuidNames) return Path.Combine(folder, $"{GetItemKey(item)}.{this.uSyncConfig.Settings.DefaultExtension}"); + var path = Path.Combine(folder, $"{this.GetItemPath(item, GuidNames, isFlat)}.{this.uSyncConfig.Settings.DefaultExtension}"); + + // if this is flat but not using GUID filenames, then we check for clashes. + if (isFlat && !GuidNames) return CheckAndFixFileClash(path, item); + return path; + } + + + /// + /// Get a clean filename that doesn't clash with any existing items. + /// + /// + /// clashes we want to resolve can occur when the safeFilename for an item + /// matches with the safe file name for something else. e.g + /// 1 Special Doc-type + /// 2 Special Doc-type + /// + /// Will both resolve to SpecialDocType.Config + /// + /// the first item to be written to disk for a clash will get the 'normal' name + /// all subsequent items will get the appended name. + /// + /// this can be completely sidestepped by using GUID filenames. + /// + virtual protected string CheckAndFixFileClash(string path, TObject item) + { + if (syncFileService.FileExists(path)) + { + var node = syncFileService.LoadXElement(path); + + if (node == null) return path; + if (GetItemKey(item) == node.GetKey()) return path; + if (GetXmlMatchString(node) == GetItemMatchString(item)) return path; + + // get here we have a clash, we should append something + var append = GetItemKey(item).ToShortKeyString(8); // (this is the shortened GUID like media folders do) + return Path.Combine(Path.GetDirectoryName(path) ?? string.Empty, + Path.GetFileNameWithoutExtension(path) + "_" + append + Path.GetExtension(path)); + } + + return path; + } + + /// + /// a string we use to match this item, with other (where there are levels) + /// + /// + /// this works because unless it's content/media you can't actually have + /// clashing aliases at different levels in the folder structure. + /// + /// So just checking the alias works, for content we overwrite these two functions. + /// + protected virtual string GetItemMatchString(TObject item) => GetItemAlias(item); + + /// + /// Calculate the matching item string from the loaded uSync XML element + /// + protected virtual string GetXmlMatchString(XElement node) => node.GetAlias(); + + /// + /// Rename an item + /// + /// + /// This doesn't get called, because renames generally are handled in the serialization because we match by key. + /// + virtual public uSyncAction Rename(TObject item) => new uSyncAction(); + + + /// + /// Group a handler belongs too (default will be settings) + /// + public virtual string Group { get; protected set; } = uSyncConstants.Groups.Settings; + + /// + /// Serialize an item to XML based on a given UDI value + /// + public SyncAttempt GetElement(Udi udi) + { + var element = FindByUdi(udi); + if (element != null) + return SerializeItem(element, new SyncSerializerOptions()); + + return SyncAttempt.Fail(udi.ToString(), ChangeType.Fail, "Item not found"); + } + + + private TObject? FindByUdi(Udi udi) + { + switch (udi) + { + case GuidUdi guidUdi: + return GetFromService(guidUdi.Guid); + case StringUdi stringUdi: + return GetFromService(stringUdi.Id); + } + + return default; + } + + /// + /// Calculate any dependencies for any given item based on loaded dependency checkers + /// + /// + /// uSync contains no dependency checkers by default - uSync.Complete will load checkers + /// when installed. + /// + public IEnumerable GetDependencies(Guid key, DependencyFlags flags) + { + if (key == Guid.Empty) + { + return GetContainerDependencies(default, flags); + } + else + { + var item = this.GetFromService(key); + if (item == null) + { + var container = this.GetContainer(key); + if (container != null) + { + return GetContainerDependencies(container, flags); + } + return Enumerable.Empty(); + } + + return GetDependencies(item, flags); + } + } + + /// + /// Calculate any dependencies for any given item based on loaded dependency checkers + /// + /// + /// uSync contains no dependency checkers by default - uSync.Complete will load checkers + /// when installed. + /// + public IEnumerable GetDependencies(int id, DependencyFlags flags) + { + // get them from the root. + if (id == -1) return GetContainerDependencies(default, flags); + + var item = this.GetFromService(id); + if (item == null) + { + var container = this.GetContainer(id); + if (container != null) + { + return GetContainerDependencies(container, flags); + } + + return Enumerable.Empty(); + } + return GetDependencies(item, flags); + } + + private bool HasDependencyCheckers() + => dependencyCheckers != null && dependencyCheckers.Count > 0; + + + /// + /// Calculate any dependencies for any given item based on loaded dependency checkers + /// + /// + /// uSync contains no dependency checkers by default - uSync.Complete will load checkers + /// when installed. + /// + protected IEnumerable GetDependencies(TObject item, DependencyFlags flags) + { + if (item == null || !HasDependencyCheckers()) return Enumerable.Empty(); + + var dependencies = new List(); + foreach (var checker in dependencyCheckers) + { + dependencies.AddRange(checker.GetDependencies(item, flags)); + } + return dependencies; + } + + /// + /// Calculate any dependencies for any given item based on loaded dependency checkers + /// + /// + /// uSync contains no dependency checkers by default - uSync.Complete will load checkers + /// when installed. + /// + private IEnumerable GetContainerDependencies(TContainer? parent, DependencyFlags flags) + { + if (!HasDependencyCheckers()) return Enumerable.Empty(); + + var dependencies = new List(); + + var containers = GetFolders(parent); + if (containers != null && containers.Any()) + { + foreach (var container in containers) + { + dependencies.AddRange(GetContainerDependencies(container, flags)); + } + } + + var children = GetChildItems(parent); + if (children != null && children.Any()) + { + foreach (var child in children) + { + var childItem = GetFromService(child); + if (childItem != null) + { + foreach (var checker in dependencyCheckers) + { + dependencies.AddRange(checker.GetDependencies(childItem, flags)); + } + } + } + } + + return dependencies.DistinctBy(x => x.Udi?.ToString() ?? x.Name).OrderByDescending(x => x.Order); + } + + #region Serializer Calls + + /// + /// call the serializer to get an items xml. + /// + protected SyncAttempt SerializeItem(TObject item, SyncSerializerOptions options) + => serializer.Serialize(item, options); + + /// + /// + /// turn the xml into an item (and optionally save it to umbraco). + /// + protected SyncAttempt DeserializeItem(XElement node, SyncSerializerOptions options) + => serializer.Deserialize(node, options); + + /// + /// perform a second pass on an item you are importing. + /// + protected SyncAttempt DeserializeItemSecondPass(TObject item, XElement node, SyncSerializerOptions options) + => serializer.DeserializeSecondPass(item, node, options); + + private SyncChangeInfo IsItemCurrent(XElement node, SyncSerializerOptions options) + { + var change = new SyncChangeInfo(); + change.CurrentNode = SerializeFromNode(node, options); + change.Change = serializer.IsCurrent(node, change.CurrentNode, options); + return change; + } + private XElement? SerializeFromNode(XElement node, SyncSerializerOptions options) + { + var item = serializer.FindItem(node); + if (item != null) + { + var cultures = node.GetCultures(); + if (!string.IsNullOrWhiteSpace(cultures)) + { + // the cultures we serialize should match any in the file. + // this means we then only check the same values at each end. + options.Settings[Core.uSyncConstants.CultureKey] = cultures; + } + + var attempt = this.SerializeItem(item, options); + if (attempt.Success) return attempt.Item; + } + + return null; + } + + private class SyncChangeInfo + { + public ChangeType Change { get; set; } + public XElement? CurrentNode { get; set; } + } + + /// + /// Find an items UDI value based on the values in the uSync XML node + /// + public Udi? FindFromNode(XElement node) + { + var item = serializer.FindItem(node); + if (item != null) + return Udi.Create(this.EntityType, serializer.ItemKey(item)); + + return null; + } + + /// + /// Calculate the current status of an item compared to the XML in a potential import + /// + public ChangeType GetItemStatus(XElement node) + { + var serializerOptions = new SyncSerializerOptions(SerializerFlags.None, this.DefaultConfig.Settings); + return this.IsItemCurrent(node, serializerOptions).Change; + } + + #endregion + + private string GetNameFromFileOrNode(string filename, XElement node) + { + if (string.IsNullOrWhiteSpace(filename) is true) return node.GetAlias(); + return syncFileService.GetSiteRelativePath(filename); + } + + + /// + /// get thekey for any caches we might call (thread based cache value) + /// + /// + protected string GetCacheKeyBase() + => $"keycache_{this.Alias}_{Thread.CurrentThread.ManagedThreadId}"; + + private string PrepCaches() + { + if (this.serializer is ISyncCachedSerializer cachedSerializer) + cachedSerializer.InitializeCache(); + + // make sure the runtime cache is clean. + var key = GetCacheKeyBase(); + + // this also clears the folder cache - as its a starts with call. + runtimeCache.ClearByKey(key); + return key; + } + + private void CleanCaches(string cacheKey) + { + runtimeCache.ClearByKey(cacheKey); + + if (this.serializer is ISyncCachedSerializer cachedSerializer) + cachedSerializer.DisposeCache(); + + } + + #region roots notifications + + /// + /// check roots isn't blocking the save + /// + public virtual void Handle(SavingNotification notification) + { + if (ShouldBlockRootChanges(notification.SavedEntities)) + { + notification.Cancel = true; + notification.Messages.Add(GetCancelMessageForRoots()); + } + } + + /// + /// check roots isn't blocking the move + /// + public virtual void Handle(MovingNotification notification) + { + if (ShouldBlockRootChanges(notification.MoveInfoCollection.Select(x => x.Entity))) + { + notification.Cancel = true; + notification.Messages.Add(GetCancelMessageForRoots()); + } + } + + /// + /// check roots isn't blocking the delete + /// + public virtual void Handle(DeletingNotification notification) + { + if (ShouldBlockRootChanges(notification.DeletedEntities)) + { + notification.Cancel = true; + notification.Messages.Add(GetCancelMessageForRoots()); + } + } + + /// + /// should we block this event based on the existance or root objects. + /// + protected bool ShouldBlockRootChanges(IEnumerable items) + { + if (!ShouldProcessEvent()) return false; + + if (uSyncConfig.Settings.LockRoot == false) return false; + + if (!HasRootFolders()) return false; + + foreach (var item in items) + { + if (RootItemExists(item)) + return true; + } + + return false; + } + + /// + /// get the message we use for cancellations + /// + protected EventMessage GetCancelMessageForRoots() + => new EventMessage("Blocked", "You cannot make this change, root level items are locked", EventMessageType.Error); + + + private bool HasRootFolders() + => syncFileService.AnyFolderExists(uSyncConfig.GetFolders()[..^1]); + + private bool RootItemExists(TObject item) + { + foreach (var folder in uSyncConfig.GetFolders()[..^1]) + { + var filename = GetPath( + Path.Combine(folder, DefaultFolder), + item, + DefaultConfig.GuidNames, + DefaultConfig.UseFlatStructure) + .ToAppSafeFileName(); + + if (syncFileService.FileExists(filename)) + return true; + + } + + return false; + } + + #endregion } \ No newline at end of file