Reduce Memory usage while creating pnp template

src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/Model/PnPInfo.cs

@@ -28,5 +28,15 @@ public class PnPInfo
         /// Defines the mapping between original file names and OpenXML file names
         /// </summary>
         public PnPFilesMap FilesMap { get; set; }
+
+        /// <summary>
+        /// Specifies whether the file streams should be used for file contenets instead of the MemoryStream.
+        /// </summary>
+        public bool UseFileStreams { get; set; } = false;
+
+        /// <summary>
+        /// Path to be used for saving file contenets instead of the MemoryStream.
+        /// </summary>
+        public string PnPFilesPath { get; set; }
     }
 }

src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackage.cs

@@ -211,7 +211,10 @@ public static PnPPackage Open(Stream stream, FileMode mode, FileAccess access)
             {
                 Package = Package.Open(stream, mode, access)
             };
-            package.EnsureMandatoryPackageComponents();
+            if (mode != FileMode.Create)
+            {
+                package.EnsureMandatoryPackageComponents();
+            }
             return package;
         }
 
@@ -228,6 +231,21 @@ public void AddFile(string fileName, Byte[] value)
             SetPackagePartValue(value, part);
         }
 
+        /// <summary>
+        /// Adds file to the package
+        /// </summary>
+        /// <param name="fileName">Name of the file</param>
+        /// <param name="stream">Stream of the file</param>
+        public void AddFilePart(string fileName, Stream stream)
+        {
+            fileName = fileName.TrimStart('/');
+            string uriStr = U_DIR_FILES + fileName;
+            // create part
+            Uri uri = GetUri(uriStr);
+            PackagePart part = Package.CreatePart(uri, CT_FILE, PACKAGE_COMPRESSION_LEVEL);
+            SetPackagePartValue(stream, part);
+        }
+
         /// <summary>
         /// Clear the files having package parts with specific relationship type
         /// </summary>
@@ -349,7 +367,7 @@ private T GetXamlSerializedPackagePartValue<T>(PackagePart part) where T : class
             return obj;
         }
 
-        private void SetXamlSerializedPackagePartValue(object value, PackagePart part)
+        static public void SetXamlSerializedPackagePartValue(object value, PackagePart part)
         {
             if (value == null)
                 return;
@@ -383,6 +401,15 @@ private void SetPackagePartValue(Byte[] value, PackagePart part)
             }
         }
 
+        private void SetPackagePartValue(Stream stream, PackagePart part)
+        {
+            using (Stream destStream = part.GetStream(FileMode.OpenOrCreate))
+            {
+                stream.Position = 0;
+                stream.CopyTo(destStream);
+            }
+        }
+
         private PackagePart CreatePackagePart(string relType, string contentType, string uriStr, PackagePart parent)
         {
             // create part & relationship

src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackageHelper.cs

@@ -4,6 +4,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.IO.Packaging;
 
 namespace PnP.Framework.Provisioning.Connectors.OpenXML
 {
@@ -28,6 +29,14 @@ public static MemoryStream PackTemplateAsStream(this PnPInfo pnpInfo)
             return stream;
         }
 
+        public static void PackTemplateToStream(this PnPInfo pnpInfo, Stream stream)
+        {
+            using (PnPPackage package = PnPPackage.Open(stream, FileMode.Create, FileAccess.Write))
+            {
+                SavePnPPackage(pnpInfo, package);
+            }
+        }
+
         /// <summary>
         /// Packs template as a stream array
         /// </summary>
@@ -113,20 +122,63 @@ private static PnPInfo LoadPnPPackage(PnPPackage package)
 
         private static void SavePnPPackage(PnPInfo pnpInfo, PnPPackage package)
         {
-            package.Manifest = pnpInfo.Manifest;
-            package.Properties = pnpInfo.Properties;
             Debug.Assert(pnpInfo.Files.TrueForAll(f => !string.IsNullOrWhiteSpace(f.InternalName)), "All files need an InternalFileName");
-            package.FilesMap = new PnPFilesMap(pnpInfo.Files.ToDictionary(f => f.InternalName, f => Path.Combine(f.Folder, f.OriginalName).Replace('\\', '/').TrimStart('/')));
-            package.ClearFiles();
-            if (pnpInfo.Files != null)
+            if (!pnpInfo.UseFileStreams)
             {
-                foreach (PnPFileInfo file in pnpInfo.Files)
+                package.Manifest = pnpInfo.Manifest;
+                package.Properties = pnpInfo.Properties;
+                package.FilesMap = new PnPFilesMap(pnpInfo.Files.ToDictionary(f => f.InternalName, f => Path.Combine(f.Folder, f.OriginalName).Replace('\\', '/').TrimStart('/')));
+                package.ClearFiles();
+                if (pnpInfo.Files != null)
                 {
-                    package.AddFile(file.InternalName, file.Content);
+                    foreach (PnPFileInfo file in pnpInfo.Files)
+                    {
+                        package.AddFile(file.InternalName, file.Content);
+                    }
                 }
             }
-        }
+            else
+            {
+                // Package with Create mode does not allow reads. Prepare and write the parts along with their relations in one go.
+                // This is a workaround for(Memory leak with Append mode) https://github.com/dotnet/runtime/issues/1544
+                var uriPath = new Uri(PnPPackage.U_PROVISIONINGTEMPLATE_MANIFEST, UriKind.Relative);
+                PackagePart manifest = package.Package.CreatePart(uriPath, PnPPackage.CT_PROVISIONINGTEMPLATE_MANIFEST, PnPPackage.PACKAGE_COMPRESSION_LEVEL);
+                PnPPackage.SetXamlSerializedPackagePartValue(pnpInfo.Manifest, manifest);
+                package.Package.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_MANIFEST);
 
+                uriPath = new Uri(PnPPackage.U_PROVISIONINGTEMPLATE_PROPERTIES, UriKind.Relative);
+                PackagePart properties = package.Package.CreatePart(uriPath, PnPPackage.CT_PROVISIONINGTEMPLATE_PROPERTIES, PnPPackage.PACKAGE_COMPRESSION_LEVEL);
+                manifest.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_PROPERTIES);
+
+                uriPath = new Uri(PnPPackage.U_FILES_ORIGIN, UriKind.Relative);
+                PackagePart filesOrigin = package.Package.CreatePart(uriPath, PnPPackage.CT_ORIGIN, PnPPackage.PACKAGE_COMPRESSION_LEVEL);
+                manifest.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_FILES_ORIGIN);
+
+                uriPath = new Uri(PnPPackage.U_PROVISIONINGTEMPLATE_FILES_MAP, UriKind.Relative);
+                PackagePart filesMap = package.Package.CreatePart(uriPath, PnPPackage.CT_PROVISIONINGTEMPLATE_FILES_MAP, PnPPackage.PACKAGE_COMPRESSION_LEVEL);
+                PnPPackage.SetXamlSerializedPackagePartValue(new PnPFilesMap(pnpInfo.Files.ToDictionary(f => f.InternalName, f => Path.Combine(f.Folder, f.OriginalName).Replace('\\', '/').TrimStart('/'))), filesMap);
+                manifest.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_FILES_MAP);
+
+                if (pnpInfo.Files != null)
+                {
+                    foreach (PnPFileInfo file in pnpInfo.Files)
+                    {
+
+#if NET6_0_OR_GREATER
+                        // Set the file stream options to delete the files automatically once closed.
+                        var fileStreamOptions = new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Options = FileOptions.DeleteOnClose, Share = FileShare.Delete };
+                        using (FileStream fs = File.Open(Path.Combine(pnpInfo.PnPFilesPath, file.InternalName).Replace('\\', '/').TrimStart('/'), fileStreamOptions))
+#else
+                        using (FileStream fs = File.OpenRead(Path.Combine(pnpInfo.PnPFilesPath, file.InternalName).Replace('\\', '/').TrimStart('/')))
+#endif
+                        {
+                            package.AddFilePart(file.InternalName, fs);
+                            filesOrigin.CreateRelationship(new Uri(PnPPackage.U_DIR_FILES + file.InternalName.TrimStart('/'), UriKind.Relative), TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_FILE);
+                        }
+                    }
+                }
+            }
+        }
         #endregion
     }
 }

src/lib/PnP.Framework/Provisioning/Connectors/OpenXMLConnector.cs

@@ -52,10 +52,12 @@ public OpenXMLConnector(Stream packageStream) : base()
         /// <param name="author">The Author of the .PNP package file, if any. Optional</param>
         /// <param name="signingCertificate">The X.509 certificate to use for digital signature of the template, optional</param>
         /// <param name="templateFileName">The name of the tempalte file, optional</param>
+        /// <param name="useFileStreams">Wheter to to use FileStream instead of MemoryStream while reading files, optional</param>
+        /// <param name="pnpFilesPath">Optional path to save files when using FileStream instead of MemoryStream while reading files, optional</param>
         public OpenXMLConnector(string packageFileName,
             FileConnectorBase persistenceConnector,
             string author = null, 
-            X509Certificate2 signingCertificate = null, string templateFileName = null)
+            X509Certificate2 signingCertificate = null, string templateFileName = null, bool useFileStreams = false, string pnpFilesPath = null)
             : base()
         {
             if (string.IsNullOrEmpty(packageFileName))
@@ -99,6 +101,8 @@ public OpenXMLConnector(string packageFileName,
                         Author = !string.IsNullOrEmpty(author) ? author : string.Empty,
                         TemplateFileName = templateFileName ?? ""
                     },
+                    UseFileStreams = useFileStreams,
+                    PnPFilesPath = useFileStreams ? (string.IsNullOrEmpty(pnpFilesPath) ? persistenceConnector.GetConnectionString() : pnpFilesPath) : string.Empty,
                 };
             }
         }
@@ -294,24 +298,48 @@ public override void SaveFileStream(string fileName, string container, Stream st
 
             try
             {
-                var memoryStream = stream.ToMemoryStream();
-                byte[] bytes = memoryStream.ToArray();
-
                 // Check if the file already exists in the package
                 var existingFile = pnpInfo.Files.FirstOrDefault(f => f.OriginalName.Equals(fileName, StringComparison.InvariantCultureIgnoreCase) && f.Folder.Equals(container, StringComparison.InvariantCultureIgnoreCase));
                 if (existingFile != null)
                 {
-                    existingFile.Content = bytes;
+                    if (pnpInfo.UseFileStreams)
+                    {
+                        using (FileStream fs = File.Create(Path.Combine(pnpInfo.PnPFilesPath, existingFile.InternalName).Replace('\\', '/').TrimStart('/')))
+                        {
+                            stream.CopyTo(fs);
+                        }
+                    }
+                    else
+                    {
+                        existingFile.Content = stream.ToMemoryStream().ToArray();
+                    }
                 }
                 else
                 {
-                    pnpInfo.Files.Add(new PnPFileInfo
+                    if (pnpInfo.UseFileStreams)
+                    {
+                        var internalFileName = fileName.AsInternalFilename();
+                        using (FileStream fs = File.Create(Path.Combine(pnpInfo.PnPFilesPath, internalFileName).Replace('\\', '/').TrimStart('/')))
+                        {
+                            stream.CopyTo(fs);
+                        }
+                        pnpInfo.Files.Add(new PnPFileInfo
+                        {
+                            InternalName = internalFileName,
+                            OriginalName = fileName,
+                            Folder = container,
+                        });
+                    }
+                    else
                     {
-                        InternalName = fileName.AsInternalFilename(),
-                        OriginalName = fileName,
-                        Folder = container,
-                        Content = bytes,
-                    });
+                        pnpInfo.Files.Add(new PnPFileInfo
+                        {
+                            InternalName = fileName.AsInternalFilename(),
+                            OriginalName = fileName,
+                            Folder = container,
+                            Content = stream.ToMemoryStream().ToArray(),
+                        });
+                    }
                 }
 
                 Log.Info(Constants.LOGGING_SOURCE, CoreResources.Provisioning_Connectors_OpenXML_FileSaved, fileName, container);
@@ -436,8 +464,20 @@ internal override string GetContainer()
         /// </summary>
         public void Commit()
         {
-            MemoryStream stream = pnpInfo.PackTemplateAsStream();
-            persistenceConnector.SaveFileStream(this.packageFileName, stream);
+            if (pnpInfo.UseFileStreams)
+            {
+                using (FileStream fs = File.Create(Path.Combine(persistenceConnector.GetConnectionString(), this.packageFileName).Replace('\\', '/').TrimStart('/')))
+                {
+                    pnpInfo.PackTemplateToStream(fs);
+                }
+            }
+            else
+            {
+                using (MemoryStream stream = pnpInfo.PackTemplateAsStream())
+                {
+                    persistenceConnector.SaveFileStream(this.packageFileName, stream);
+                }
+            }
         }
 
         #endregion

src/lib/PnP.Framework/Provisioning/Connectors/SharePointConnector.cs

@@ -440,6 +440,7 @@ private MemoryStream GetFileFromStorage(string fileName, string container)
                         cc.ExecuteQueryRetry();
 
                         streamResult.Value.CopyTo(stream);
+                        streamResult.Value.Dispose();
 
                         Log.Info(Constants.LOGGING_SOURCE, CoreResources.Provisioning_Connectors_SharePoint_FileRetrieved, fileName, GetConnectionString(), container);
 

src/lib/PnP.Framework/Provisioning/Providers/Xml/XMLTemplateProvider.cs

@@ -259,13 +259,14 @@ public override void SaveAs(ProvisioningHierarchy hierarchy, string uri, ITempla
             }
             formatter.Initialize(this);
 
-            var stream = ((IProvisioningHierarchyFormatter)formatter).ToFormattedHierarchy(hierarchy);
-
-            this.Connector.SaveFileStream(uri, stream);
-
-            if (this.Connector is ICommitableFileConnector)
+            using (var stream = ((IProvisioningHierarchyFormatter)formatter).ToFormattedHierarchy(hierarchy))
             {
-                ((ICommitableFileConnector)this.Connector).Commit();
+                this.Connector.SaveFileStream(uri, stream);
+
+                if (this.Connector is ICommitableFileConnector)
+                {
+                    ((ICommitableFileConnector)this.Connector).Commit();
+                }
             }
         }