diff --git a/src/lib/PnP.Framework/Provisioning/Connectors/FileSystemConnector.cs b/src/lib/PnP.Framework/Provisioning/Connectors/FileSystemConnector.cs index 1ac0f61cc..e7ff8df39 100644 --- a/src/lib/PnP.Framework/Provisioning/Connectors/FileSystemConnector.cs +++ b/src/lib/PnP.Framework/Provisioning/Connectors/FileSystemConnector.cs @@ -12,6 +12,8 @@ namespace PnP.Framework.Provisioning.Connectors /// public class FileSystemConnector : FileConnectorBase { + private readonly bool useFileStreams; + #region Constructors /// /// Base constructor @@ -27,7 +29,8 @@ public FileSystemConnector() /// /// Root folder (e.g. c:\temp or .\resources or . or .\resources\templates) /// Sub folder (e.g. templates or resources\templates or blank - public FileSystemConnector(string connectionString, string container) + /// Use FileStreams instead of MemoryStreams + public FileSystemConnector(string connectionString, string container, bool useFileStreams = false) : base() { if (String.IsNullOrEmpty(connectionString)) @@ -41,6 +44,7 @@ public FileSystemConnector(string connectionString, string container) } container = container.Replace('/', '\\'); + this.useFileStreams = useFileStreams; this.AddParameterAsString(CONNECTIONSTRING, connectionString); this.AddParameterAsString(CONTAINER, container); } @@ -208,6 +212,28 @@ public override Stream GetFileStream(string fileName, string container) } container = container.Replace('/', '\\'); + if (useFileStreams) + { + try + { + string filePath = ConstructPath(fileName, container); + FileStream fileStream = File.OpenRead(filePath); + Log.Info(Constants.LOGGING_SOURCE, CoreResources.Provisioning_Connectors_FileSystem_FileRetrieved, fileName, container); + fileStream.Position = 0; + return fileStream; + } + catch (Exception ex) + { + if (ex is FileNotFoundException || ex is DirectoryNotFoundException) + { + Log.Error(Constants.LOGGING_SOURCE, CoreResources.Provisioning_Connectors_FileSystem_FileNotFound, fileName, container, ex.Message); + return null; + } + + throw; + } + } + return GetFileFromStorage(fileName, container); } diff --git a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/Model/PnPInfo.cs b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/Model/PnPInfo.cs index c0946d019..69180333e 100644 --- a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/Model/PnPInfo.cs +++ b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/Model/PnPInfo.cs @@ -28,5 +28,15 @@ public class PnPInfo /// Defines the mapping between original file names and OpenXML file names /// public PnPFilesMap FilesMap { get; set; } + + /// + /// Specifies whether the file streams should be used for file contenets instead of the MemoryStream. + /// + public bool UseFileStreams { get; set; } = false; + + /// + /// Path to be used for saving file contenets instead of the MemoryStream. + /// + public string PnPFilesPath { get; set; } } } diff --git a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackage.cs b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackage.cs index dcf2dfb5a..3fb63fd5e 100644 --- a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackage.cs +++ b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackage.cs @@ -50,6 +50,16 @@ public partial class PnPPackage : IDisposable #region Public Properties + /// + /// Specifies whether the file streams should be used for file contenets instead of the MemoryStream. + /// + public bool UseFileStreams { get; set; } = false; + + /// + /// Path to be used for saving file contenets instead of the MemoryStream. + /// + public string PnPFilesPath { get; set; } + /// /// The complete package object /// @@ -164,6 +174,25 @@ public IDictionary Files originalName.Substring(0, originalName.LastIndexOf('/')) : string.Empty; } + if (UseFileStreams && p != null) + { + using (Stream stream = p.GetStream()) + { + using (FileStream fs = File.Create(Path.Combine(PnPFilesPath, fileName).Replace('\\', '/').TrimStart('/'))) + { + stream.CopyTo(fs); + } + } + + result[fileName] = new PnPPackageFileItem + { + Name = fileName, + Folder = folder, + }; + + continue; + } + Byte[] content = ReadPackagePartBytes(p); result[fileName] = new PnPPackageFileItem @@ -211,7 +240,10 @@ public static PnPPackage Open(Stream stream, FileMode mode, FileAccess access) { Package = Package.Open(stream, mode, access) }; - package.EnsureMandatoryPackageComponents(); + if (mode != FileMode.Create) + { + package.EnsureMandatoryPackageComponents(); + } return package; } @@ -228,6 +260,21 @@ public void AddFile(string fileName, Byte[] value) SetPackagePartValue(value, part); } + /// + /// Adds file to the package + /// + /// Name of the file + /// Stream of the file + public void AddFilePart(string fileName, Stream stream) + { + fileName = fileName.TrimStart('/'); + string uriStr = U_DIR_FILES + fileName; + // create part + Uri uri = GetUri(uriStr); + PackagePart part = Package.CreatePart(uri, CT_FILE, PACKAGE_COMPRESSION_LEVEL); + SetPackagePartValue(stream, part); + } + /// /// Clear the files having package parts with specific relationship type /// @@ -337,19 +384,22 @@ private T GetXamlSerializedPackagePartValue(PackagePart part) where T : class } else { - stream.Seek(0, SeekOrigin.Begin); - if (stream.Length == 0) - { + if (string.IsNullOrEmpty(textContent)) { return null; } - obj = (T)XamlServices.Load(stream); + + var contentBytes = System.Text.Encoding.UTF8.GetBytes(textContent); + using (var memoryStream = new MemoryStream(contentBytes)) + { + obj = (T)XamlServices.Load(memoryStream); + } } } } return obj; } - private void SetXamlSerializedPackagePartValue(object value, PackagePart part) + static public void SetXamlSerializedPackagePartValue(object value, PackagePart part) { if (value == null) return; @@ -383,6 +433,15 @@ private void SetPackagePartValue(Byte[] value, PackagePart part) } } + private void SetPackagePartValue(Stream stream, PackagePart part) + { + using (Stream destStream = part.GetStream(FileMode.OpenOrCreate)) + { + stream.Position = 0; + stream.CopyTo(destStream); + } + } + private PackagePart CreatePackagePart(string relType, string contentType, string uriStr, PackagePart parent) { // create part & relationship diff --git a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackageHelper.cs b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackageHelper.cs index 4b8e28875..e6b2ceafc 100644 --- a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackageHelper.cs +++ b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackageHelper.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.IO.Packaging; namespace PnP.Framework.Provisioning.Connectors.OpenXML { @@ -28,6 +29,14 @@ public static MemoryStream PackTemplateAsStream(this PnPInfo pnpInfo) return stream; } + public static void PackTemplateToStream(this PnPInfo pnpInfo, Stream stream) + { + using (PnPPackage package = PnPPackage.Open(stream, FileMode.Create, FileAccess.Write)) + { + SavePnPPackage(pnpInfo, package); + } + } + /// /// Packs template as a stream array /// @@ -41,6 +50,27 @@ public static Byte[] PackTemplate(this PnPInfo pnpInfo) } } + /// + /// Unpacks template into PnP OpenXML package info object based on file stream + /// + /// Stream + /// Use FileStreams + /// Temp pnp Files Path + /// Returns site template + public static PnPInfo UnpackTemplate(this Stream stream, bool useFileStreams = false, string pnpFilesPath = null) + { + PnPInfo siteTemplate; + using (PnPPackage package = PnPPackage.Open(stream, FileMode.Open, FileAccess.Read)) + { + if (useFileStreams) { + package.UseFileStreams = useFileStreams; + package.PnPFilesPath = pnpFilesPath; + } + siteTemplate = LoadPnPPackage(package); + } + return siteTemplate; + } + /// /// Unpacks template into PnP OpenXML package info object based on memory stream /// @@ -89,6 +119,8 @@ private static PnPInfo LoadPnPPackage(PnPPackage package) Manifest = package.Manifest, Properties = package.Properties, FilesMap = package.FilesMap, + UseFileStreams = package.UseFileStreams, + PnPFilesPath = package.PnPFilesPath, Files = new List() }; @@ -113,20 +145,63 @@ private static PnPInfo LoadPnPPackage(PnPPackage package) private static void SavePnPPackage(PnPInfo pnpInfo, PnPPackage package) { - package.Manifest = pnpInfo.Manifest; - package.Properties = pnpInfo.Properties; Debug.Assert(pnpInfo.Files.TrueForAll(f => !string.IsNullOrWhiteSpace(f.InternalName)), "All files need an InternalFileName"); - package.FilesMap = new PnPFilesMap(pnpInfo.Files.ToDictionary(f => f.InternalName, f => Path.Combine(f.Folder, f.OriginalName).Replace('\\', '/').TrimStart('/'))); - package.ClearFiles(); - if (pnpInfo.Files != null) + if (!pnpInfo.UseFileStreams) { - foreach (PnPFileInfo file in pnpInfo.Files) + package.Manifest = pnpInfo.Manifest; + package.Properties = pnpInfo.Properties; + package.FilesMap = new PnPFilesMap(pnpInfo.Files.ToDictionary(f => f.InternalName, f => Path.Combine(f.Folder, f.OriginalName).Replace('\\', '/').TrimStart('/'))); + package.ClearFiles(); + if (pnpInfo.Files != null) { - package.AddFile(file.InternalName, file.Content); + foreach (PnPFileInfo file in pnpInfo.Files) + { + package.AddFile(file.InternalName, file.Content); + } } } - } + else + { + // Package with Create mode does not allow reads. Prepare and write the parts along with their relations in one go. + // This is a workaround for(Memory leak with Append mode) https://github.com/dotnet/runtime/issues/1544 + var uriPath = new Uri(PnPPackage.U_PROVISIONINGTEMPLATE_MANIFEST, UriKind.Relative); + PackagePart manifest = package.Package.CreatePart(uriPath, PnPPackage.CT_PROVISIONINGTEMPLATE_MANIFEST, PnPPackage.PACKAGE_COMPRESSION_LEVEL); + PnPPackage.SetXamlSerializedPackagePartValue(pnpInfo.Manifest, manifest); + package.Package.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_MANIFEST); + + uriPath = new Uri(PnPPackage.U_PROVISIONINGTEMPLATE_PROPERTIES, UriKind.Relative); + PackagePart properties = package.Package.CreatePart(uriPath, PnPPackage.CT_PROVISIONINGTEMPLATE_PROPERTIES, PnPPackage.PACKAGE_COMPRESSION_LEVEL); + manifest.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_PROPERTIES); + + uriPath = new Uri(PnPPackage.U_FILES_ORIGIN, UriKind.Relative); + PackagePart filesOrigin = package.Package.CreatePart(uriPath, PnPPackage.CT_ORIGIN, PnPPackage.PACKAGE_COMPRESSION_LEVEL); + manifest.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_FILES_ORIGIN); + uriPath = new Uri(PnPPackage.U_PROVISIONINGTEMPLATE_FILES_MAP, UriKind.Relative); + PackagePart filesMap = package.Package.CreatePart(uriPath, PnPPackage.CT_PROVISIONINGTEMPLATE_FILES_MAP, PnPPackage.PACKAGE_COMPRESSION_LEVEL); + PnPPackage.SetXamlSerializedPackagePartValue(new PnPFilesMap(pnpInfo.Files.ToDictionary(f => f.InternalName, f => Path.Combine(f.Folder, f.OriginalName).Replace('\\', '/').TrimStart('/'))), filesMap); + manifest.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_FILES_MAP); + + if (pnpInfo.Files != null) + { + foreach (PnPFileInfo file in pnpInfo.Files) + { + +#if NET6_0_OR_GREATER + // Set the file stream options to delete the files automatically once closed. + var fileStreamOptions = new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Options = FileOptions.DeleteOnClose, Share = FileShare.Delete }; + using (FileStream fs = File.Open(Path.Combine(pnpInfo.PnPFilesPath, file.InternalName).Replace('\\', '/').TrimStart('/'), fileStreamOptions)) +#else + using (FileStream fs = File.OpenRead(Path.Combine(pnpInfo.PnPFilesPath, file.InternalName).Replace('\\', '/').TrimStart('/'))) +#endif + { + package.AddFilePart(file.InternalName, fs); + filesOrigin.CreateRelationship(new Uri(PnPPackage.U_DIR_FILES + file.InternalName.TrimStart('/'), UriKind.Relative), TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_FILE); + } + } + } + } + } #endregion } } diff --git a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXMLConnector.cs b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXMLConnector.cs index fe04ca5fe..e7318cb69 100644 --- a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXMLConnector.cs +++ b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXMLConnector.cs @@ -52,10 +52,12 @@ public OpenXMLConnector(Stream packageStream) : base() /// The Author of the .PNP package file, if any. Optional /// The X.509 certificate to use for digital signature of the template, optional /// The name of the tempalte file, optional + /// Wheter to to use FileStream instead of MemoryStream while reading files, optional + /// Optional path to save files when using FileStream instead of MemoryStream while reading files, optional public OpenXMLConnector(string packageFileName, FileConnectorBase persistenceConnector, string author = null, - X509Certificate2 signingCertificate = null, string templateFileName = null) + X509Certificate2 signingCertificate = null, string templateFileName = null, bool useFileStreams = false, string pnpFilesPath = null) : base() { if (string.IsNullOrEmpty(packageFileName)) @@ -81,8 +83,15 @@ public OpenXMLConnector(string packageFileName, if (packageStream != null) { // If the .PNP package exists unpack it into PnP OpenXML package info object - MemoryStream ms = packageStream.ToMemoryStream(); - this.pnpInfo = ms.UnpackTemplate(); + if (!useFileStreams) + { + MemoryStream ms = packageStream.ToMemoryStream(); + this.pnpInfo = ms.UnpackTemplate(); + } + else + { + this.pnpInfo = packageStream.UnpackTemplate(useFileStreams, useFileStreams ? (string.IsNullOrEmpty(pnpFilesPath) ? persistenceConnector.GetConnectionString() : pnpFilesPath) : string.Empty); + } } else { @@ -99,6 +108,8 @@ public OpenXMLConnector(string packageFileName, Author = !string.IsNullOrEmpty(author) ? author : string.Empty, TemplateFileName = templateFileName ?? "" }, + UseFileStreams = useFileStreams, + PnPFilesPath = useFileStreams ? (string.IsNullOrEmpty(pnpFilesPath) ? persistenceConnector.GetConnectionString() : pnpFilesPath) : string.Empty, }; } } @@ -254,7 +265,42 @@ public override Stream GetFileStream(string fileName, string container) container = ""; } - return GetFileFromStorage(fileName, container); + if (!pnpInfo.UseFileStreams) + { + return GetFileFromStorage(fileName, container); + } + + try + { + var file = GetFileFromInsidePackage(fileName, container); + + if (file != null) + { + Log.Info(Constants.LOGGING_SOURCE, CoreResources.Provisioning_Connectors_OpenXML_FileRetrieved, fileName, container); +#if NET6_0_OR_GREATER + // Set the file stream options to delete the file automatically once closed. + var fileStreamOptions = new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Options = FileOptions.DeleteOnClose, Share = FileShare.Delete }; + FileStream fs = File.Open(Path.Combine(pnpInfo.PnPFilesPath, file.InternalName).Replace('\\', '/').TrimStart('/'), fileStreamOptions); +#else + FileStream fs = File.OpenRead(Path.Combine(pnpInfo.PnPFilesPath, file.InternalName).Replace('\\', '/').TrimStart('/')); +#endif + return fs; + } + else + { + throw new FileNotFoundException(); + } + } + catch (Exception ex) + { + if (ex is FileNotFoundException || ex is DirectoryNotFoundException) + { + Log.Error(Constants.LOGGING_SOURCE, CoreResources.Provisioning_Connectors_OpenXML_FileNotFound, fileName, container, ex.Message); + return null; + } + + throw; + } } /// @@ -294,24 +340,48 @@ public override void SaveFileStream(string fileName, string container, Stream st try { - var memoryStream = stream.ToMemoryStream(); - byte[] bytes = memoryStream.ToArray(); - // Check if the file already exists in the package var existingFile = pnpInfo.Files.FirstOrDefault(f => f.OriginalName.Equals(fileName, StringComparison.InvariantCultureIgnoreCase) && f.Folder.Equals(container, StringComparison.InvariantCultureIgnoreCase)); if (existingFile != null) { - existingFile.Content = bytes; + if (pnpInfo.UseFileStreams) + { + using (FileStream fs = File.Create(Path.Combine(pnpInfo.PnPFilesPath, existingFile.InternalName).Replace('\\', '/').TrimStart('/'))) + { + stream.CopyTo(fs); + } + } + else + { + existingFile.Content = stream.ToMemoryStream().ToArray(); + } } else { - pnpInfo.Files.Add(new PnPFileInfo + if (pnpInfo.UseFileStreams) + { + var internalFileName = fileName.AsInternalFilename(); + using (FileStream fs = File.Create(Path.Combine(pnpInfo.PnPFilesPath, internalFileName).Replace('\\', '/').TrimStart('/'))) + { + stream.CopyTo(fs); + } + pnpInfo.Files.Add(new PnPFileInfo + { + InternalName = internalFileName, + OriginalName = fileName, + Folder = container, + }); + } + else { - InternalName = fileName.AsInternalFilename(), - OriginalName = fileName, - Folder = container, - Content = bytes, - }); + pnpInfo.Files.Add(new PnPFileInfo + { + InternalName = fileName.AsInternalFilename(), + OriginalName = fileName, + Folder = container, + Content = stream.ToMemoryStream().ToArray(), + }); + } } Log.Info(Constants.LOGGING_SOURCE, CoreResources.Provisioning_Connectors_OpenXML_FileSaved, fileName, container); @@ -380,6 +450,15 @@ private MemoryStream GetFileFromStorage(string fileName, string container) if (file != null) { Log.Info(Constants.LOGGING_SOURCE, CoreResources.Provisioning_Connectors_OpenXML_FileRetrieved, fileName, container); + + if (pnpInfo.UseFileStreams) + { + using (FileStream fs = File.OpenRead(Path.Combine(pnpInfo.PnPFilesPath, file.InternalName).Replace('\\', '/').TrimStart('/'))) + { + return fs.ToMemoryStream(); + } + } + var stream = new MemoryStream(file.Content); return stream; } @@ -436,8 +515,20 @@ internal override string GetContainer() /// public void Commit() { - MemoryStream stream = pnpInfo.PackTemplateAsStream(); - persistenceConnector.SaveFileStream(this.packageFileName, stream); + if (pnpInfo.UseFileStreams) + { + using (FileStream fs = File.Create(Path.Combine(persistenceConnector.GetConnectionString(), this.packageFileName).Replace('\\', '/').TrimStart('/'))) + { + pnpInfo.PackTemplateToStream(fs); + } + } + else + { + using (MemoryStream stream = pnpInfo.PackTemplateAsStream()) + { + persistenceConnector.SaveFileStream(this.packageFileName, stream); + } + } } #endregion diff --git a/src/lib/PnP.Framework/Provisioning/Connectors/SharePointConnector.cs b/src/lib/PnP.Framework/Provisioning/Connectors/SharePointConnector.cs index 942b34ef3..0fc7aa02f 100644 --- a/src/lib/PnP.Framework/Provisioning/Connectors/SharePointConnector.cs +++ b/src/lib/PnP.Framework/Provisioning/Connectors/SharePointConnector.cs @@ -440,6 +440,7 @@ private MemoryStream GetFileFromStorage(string fileName, string container) cc.ExecuteQueryRetry(); streamResult.Value.CopyTo(stream); + streamResult.Value.Dispose(); Log.Info(Constants.LOGGING_SOURCE, CoreResources.Provisioning_Connectors_SharePoint_FileRetrieved, fileName, GetConnectionString(), container); diff --git a/src/lib/PnP.Framework/Provisioning/Providers/Xml/XMLTemplateProvider.cs b/src/lib/PnP.Framework/Provisioning/Providers/Xml/XMLTemplateProvider.cs index 6e96a1688..99e0c2716 100644 --- a/src/lib/PnP.Framework/Provisioning/Providers/Xml/XMLTemplateProvider.cs +++ b/src/lib/PnP.Framework/Provisioning/Providers/Xml/XMLTemplateProvider.cs @@ -259,13 +259,14 @@ public override void SaveAs(ProvisioningHierarchy hierarchy, string uri, ITempla } formatter.Initialize(this); - var stream = ((IProvisioningHierarchyFormatter)formatter).ToFormattedHierarchy(hierarchy); - - this.Connector.SaveFileStream(uri, stream); - - if (this.Connector is ICommitableFileConnector) + using (var stream = ((IProvisioningHierarchyFormatter)formatter).ToFormattedHierarchy(hierarchy)) { - ((ICommitableFileConnector)this.Connector).Commit(); + this.Connector.SaveFileStream(uri, stream); + + if (this.Connector is ICommitableFileConnector) + { + ((ICommitableFileConnector)this.Connector).Commit(); + } } }