Reduce Memory usage while applying pnp template

src/lib/PnP.Framework/Provisioning/Connectors/FileSystemConnector.cs

@@ -12,6 +12,8 @@ namespace PnP.Framework.Provisioning.Connectors
     /// </summary>
     public class FileSystemConnector : FileConnectorBase
     {
+        private readonly bool useFileStreams;
+
         #region Constructors
         /// <summary>
         /// Base constructor
@@ -27,7 +29,8 @@ public FileSystemConnector()
         /// </summary>
         /// <param name="connectionString">Root folder (e.g. c:\temp or .\resources or . or .\resources\templates)</param>
         /// <param name="container">Sub folder (e.g. templates or resources\templates or blank</param>
-        public FileSystemConnector(string connectionString, string container)
+        /// <param name="useFileStreams">Use FileStreams instead of MemoryStreams</param>
+        public FileSystemConnector(string connectionString, string container, bool useFileStreams = false)
             : base()
         {
             if (String.IsNullOrEmpty(connectionString))
@@ -41,6 +44,7 @@ public FileSystemConnector(string connectionString, string container)
             }
             container = container.Replace('/', '\\');
 
+            this.useFileStreams = useFileStreams;
             this.AddParameterAsString(CONNECTIONSTRING, connectionString);
             this.AddParameterAsString(CONTAINER, container);
         }
@@ -208,6 +212,28 @@ public override Stream GetFileStream(string fileName, string container)
             }
             container = container.Replace('/', '\\');
 
+            if (useFileStreams)
+            {
+                try
+                {
+                    string filePath = ConstructPath(fileName, container);
+                    FileStream fileStream = File.OpenRead(filePath);
+                    Log.Info(Constants.LOGGING_SOURCE, CoreResources.Provisioning_Connectors_FileSystem_FileRetrieved, fileName, container);
+                    fileStream.Position = 0;
+                    return fileStream;
+                }
+                catch (Exception ex)
+                {
+                    if (ex is FileNotFoundException || ex is DirectoryNotFoundException)
+                    {
+                        Log.Error(Constants.LOGGING_SOURCE, CoreResources.Provisioning_Connectors_FileSystem_FileNotFound, fileName, container, ex.Message);
+                        return null;
+                    }
+
+                    throw;
+                }
+            }
+
             return GetFileFromStorage(fileName, container);
         }
 

src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/Model/PnPInfo.cs

@@ -28,5 +28,15 @@ public class PnPInfo
         /// Defines the mapping between original file names and OpenXML file names
         /// </summary>
         public PnPFilesMap FilesMap { get; set; }
+
+        /// <summary>
+        /// Specifies whether the file streams should be used for file contenets instead of the MemoryStream.
+        /// </summary>
+        public bool UseFileStreams { get; set; } = false;
+
+        /// <summary>
+        /// Path to be used for saving file contenets instead of the MemoryStream.
+        /// </summary>
+        public string PnPFilesPath { get; set; }
     }
 }

src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackage.cs

@@ -50,6 +50,16 @@ public partial class PnPPackage : IDisposable
 
         #region Public Properties
 
+        /// <summary>
+        /// Specifies whether the file streams should be used for file contenets instead of the MemoryStream.
+        /// </summary>
+        public bool UseFileStreams { get; set; } = false;
+
+        /// <summary>
+        /// Path to be used for saving file contenets instead of the MemoryStream.
+        /// </summary>
+        public string PnPFilesPath { get; set; }
+
         /// <summary>
         /// The complete package object
         /// </summary>
@@ -164,6 +174,25 @@ public IDictionary<String, PnPPackageFileItem> Files
                             originalName.Substring(0, originalName.LastIndexOf('/')) : string.Empty;
                     }
 
+                    if (UseFileStreams && p != null)
+                    {
+                        using (Stream stream = p.GetStream())
+                        {
+                            using (FileStream fs = File.Create(Path.Combine(PnPFilesPath, fileName).Replace('\\', '/').TrimStart('/')))
+                            {
+                                stream.CopyTo(fs);
+                            }
+                        }
+
+                        result[fileName] = new PnPPackageFileItem
+                        {
+                            Name = fileName,
+                            Folder = folder,
+                        };
+
+                        continue;
+                    }
+
                     Byte[] content = ReadPackagePartBytes(p);
 
                     result[fileName] = new PnPPackageFileItem
@@ -211,7 +240,10 @@ public static PnPPackage Open(Stream stream, FileMode mode, FileAccess access)
             {
                 Package = Package.Open(stream, mode, access)
             };
-            package.EnsureMandatoryPackageComponents();
+            if (mode != FileMode.Create)
+            {
+                package.EnsureMandatoryPackageComponents();
+            }
             return package;
         }
 
@@ -228,6 +260,21 @@ public void AddFile(string fileName, Byte[] value)
             SetPackagePartValue(value, part);
         }
 
+        /// <summary>
+        /// Adds file to the package
+        /// </summary>
+        /// <param name="fileName">Name of the file</param>
+        /// <param name="stream">Stream of the file</param>
+        public void AddFilePart(string fileName, Stream stream)
+        {
+            fileName = fileName.TrimStart('/');
+            string uriStr = U_DIR_FILES + fileName;
+            // create part
+            Uri uri = GetUri(uriStr);
+            PackagePart part = Package.CreatePart(uri, CT_FILE, PACKAGE_COMPRESSION_LEVEL);
+            SetPackagePartValue(stream, part);
+        }
+
         /// <summary>
         /// Clear the files having package parts with specific relationship type
         /// </summary>
@@ -337,19 +384,22 @@ private T GetXamlSerializedPackagePartValue<T>(PackagePart part) where T : class
                     }
                     else
                     {
-                        stream.Seek(0, SeekOrigin.Begin);
-                        if (stream.Length == 0)
-                        {
+                        if (string.IsNullOrEmpty(textContent)) {
                             return null;
                         }
-                        obj = (T)XamlServices.Load(stream);
+
+                        var contentBytes = System.Text.Encoding.UTF8.GetBytes(textContent);
+                        using (var memoryStream = new MemoryStream(contentBytes))
+                        {
+                            obj = (T)XamlServices.Load(memoryStream);
+                        }
                     }
                 }
             }
             return obj;
         }
 
-        private void SetXamlSerializedPackagePartValue(object value, PackagePart part)
+        static public void SetXamlSerializedPackagePartValue(object value, PackagePart part)
         {
             if (value == null)
                 return;
@@ -383,6 +433,15 @@ private void SetPackagePartValue(Byte[] value, PackagePart part)
             }
         }
 
+        private void SetPackagePartValue(Stream stream, PackagePart part)
+        {
+            using (Stream destStream = part.GetStream(FileMode.OpenOrCreate))
+            {
+                stream.Position = 0;
+                stream.CopyTo(destStream);
+            }
+        }
+
         private PackagePart CreatePackagePart(string relType, string contentType, string uriStr, PackagePart parent)
         {
             // create part & relationship

src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackageHelper.cs

@@ -4,6 +4,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.IO.Packaging;
 
 namespace PnP.Framework.Provisioning.Connectors.OpenXML
 {
@@ -28,6 +29,14 @@ public static MemoryStream PackTemplateAsStream(this PnPInfo pnpInfo)
             return stream;
         }
 
+        public static void PackTemplateToStream(this PnPInfo pnpInfo, Stream stream)
+        {
+            using (PnPPackage package = PnPPackage.Open(stream, FileMode.Create, FileAccess.Write))
+            {
+                SavePnPPackage(pnpInfo, package);
+            }
+        }
+
         /// <summary>
         /// Packs template as a stream array
         /// </summary>
@@ -41,6 +50,27 @@ public static Byte[] PackTemplate(this PnPInfo pnpInfo)
             }
         }
 
+        /// <summary>
+        /// Unpacks template into PnP OpenXML package info object based on file stream
+        /// </summary>
+        /// <param name="stream">Stream</param>
+        /// <param name="useFileStreams">Use FileStreams</param>
+        /// <param name="pnpFilesPath">Temp pnp Files Path</param>
+        /// <returns>Returns site template</returns>
+        public static PnPInfo UnpackTemplate(this Stream stream, bool useFileStreams = false, string pnpFilesPath = null)
+        {
+            PnPInfo siteTemplate;
+            using (PnPPackage package = PnPPackage.Open(stream, FileMode.Open, FileAccess.Read))
+            {
+                if (useFileStreams) {
+                    package.UseFileStreams = useFileStreams;
+                    package.PnPFilesPath = pnpFilesPath;
+                }
+                siteTemplate = LoadPnPPackage(package);
+            }
+            return siteTemplate;
+        }
+
         /// <summary>
         /// Unpacks template into PnP OpenXML package info object based on memory stream
         /// </summary>
@@ -89,6 +119,8 @@ private static PnPInfo LoadPnPPackage(PnPPackage package)
                 Manifest = package.Manifest,
                 Properties = package.Properties,
                 FilesMap = package.FilesMap,
+                UseFileStreams = package.UseFileStreams,
+                PnPFilesPath = package.PnPFilesPath,
 
                 Files = new List<PnPFileInfo>()
             };
@@ -113,20 +145,63 @@ private static PnPInfo LoadPnPPackage(PnPPackage package)
 
         private static void SavePnPPackage(PnPInfo pnpInfo, PnPPackage package)
         {
-            package.Manifest = pnpInfo.Manifest;
-            package.Properties = pnpInfo.Properties;
             Debug.Assert(pnpInfo.Files.TrueForAll(f => !string.IsNullOrWhiteSpace(f.InternalName)), "All files need an InternalFileName");
-            package.FilesMap = new PnPFilesMap(pnpInfo.Files.ToDictionary(f => f.InternalName, f => Path.Combine(f.Folder, f.OriginalName).Replace('\\', '/').TrimStart('/')));
-            package.ClearFiles();
-            if (pnpInfo.Files != null)
+            if (!pnpInfo.UseFileStreams)
             {
-                foreach (PnPFileInfo file in pnpInfo.Files)
+                package.Manifest = pnpInfo.Manifest;
+                package.Properties = pnpInfo.Properties;
+                package.FilesMap = new PnPFilesMap(pnpInfo.Files.ToDictionary(f => f.InternalName, f => Path.Combine(f.Folder, f.OriginalName).Replace('\\', '/').TrimStart('/')));
+                package.ClearFiles();
+                if (pnpInfo.Files != null)
                 {
-                    package.AddFile(file.InternalName, file.Content);
+                    foreach (PnPFileInfo file in pnpInfo.Files)
+                    {
+                        package.AddFile(file.InternalName, file.Content);
+                    }
                 }
             }
-        }
+            else
+            {
+                // Package with Create mode does not allow reads. Prepare and write the parts along with their relations in one go.
+                // This is a workaround for(Memory leak with Append mode) https://github.com/dotnet/runtime/issues/1544
+                var uriPath = new Uri(PnPPackage.U_PROVISIONINGTEMPLATE_MANIFEST, UriKind.Relative);
+                PackagePart manifest = package.Package.CreatePart(uriPath, PnPPackage.CT_PROVISIONINGTEMPLATE_MANIFEST, PnPPackage.PACKAGE_COMPRESSION_LEVEL);
+                PnPPackage.SetXamlSerializedPackagePartValue(pnpInfo.Manifest, manifest);
+                package.Package.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_MANIFEST);
+
+                uriPath = new Uri(PnPPackage.U_PROVISIONINGTEMPLATE_PROPERTIES, UriKind.Relative);
+                PackagePart properties = package.Package.CreatePart(uriPath, PnPPackage.CT_PROVISIONINGTEMPLATE_PROPERTIES, PnPPackage.PACKAGE_COMPRESSION_LEVEL);
+                manifest.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_PROPERTIES);
+
+                uriPath = new Uri(PnPPackage.U_FILES_ORIGIN, UriKind.Relative);
+                PackagePart filesOrigin = package.Package.CreatePart(uriPath, PnPPackage.CT_ORIGIN, PnPPackage.PACKAGE_COMPRESSION_LEVEL);
+                manifest.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_FILES_ORIGIN);
 
+                uriPath = new Uri(PnPPackage.U_PROVISIONINGTEMPLATE_FILES_MAP, UriKind.Relative);
+                PackagePart filesMap = package.Package.CreatePart(uriPath, PnPPackage.CT_PROVISIONINGTEMPLATE_FILES_MAP, PnPPackage.PACKAGE_COMPRESSION_LEVEL);
+                PnPPackage.SetXamlSerializedPackagePartValue(new PnPFilesMap(pnpInfo.Files.ToDictionary(f => f.InternalName, f => Path.Combine(f.Folder, f.OriginalName).Replace('\\', '/').TrimStart('/'))), filesMap);
+                manifest.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_FILES_MAP);
+
+                if (pnpInfo.Files != null)
+                {
+                    foreach (PnPFileInfo file in pnpInfo.Files)
+                    {
+
+#if NET6_0_OR_GREATER
+                        // Set the file stream options to delete the files automatically once closed.
+                        var fileStreamOptions = new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Options = FileOptions.DeleteOnClose, Share = FileShare.Delete };
+                        using (FileStream fs = File.Open(Path.Combine(pnpInfo.PnPFilesPath, file.InternalName).Replace('\\', '/').TrimStart('/'), fileStreamOptions))
+#else
+                        using (FileStream fs = File.OpenRead(Path.Combine(pnpInfo.PnPFilesPath, file.InternalName).Replace('\\', '/').TrimStart('/')))
+#endif
+                        {
+                            package.AddFilePart(file.InternalName, fs);
+                            filesOrigin.CreateRelationship(new Uri(PnPPackage.U_DIR_FILES + file.InternalName.TrimStart('/'), UriKind.Relative), TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_FILE);
+                        }
+                    }
+                }
+            }
+        }
         #endregion
     }
 }