Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimizing file upload speed by reducing the number of API calls #764

Draft
wants to merge 1 commit into
base: dev
Choose a base branch
from
+43 −14
Draft

Optimizing file upload speed by reducing the number of API calls #764

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 43 additions & 14 deletions src/lib/PnP.Framework/Provisioning/ObjectHandlers/ObjectFiles.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Net;
using File = Microsoft.SharePoint.Client.File;

Expand Down Expand Up @@ -57,6 +58,8 @@ public override TokenParser ProvisionObjects(Web web, ProvisioningTemplate templ

var currentFileIndex = 0;
var originalWeb = web; // Used to store and re-store context in case files are deployed to masterpage gallery
// PERFORMANCE NOTE: save already retrieved folder info to speed up uploading files to the same folders
var knownFolders = new Dictionary<string, Microsoft.SharePoint.Client.Folder>();
foreach (var file in filesToProcess)
{
file.Src = parser.ParseString(file.Src);
Expand Down Expand Up @@ -88,16 +91,27 @@ public override TokenParser ProvisionObjects(Web web, ProvisioningTemplate templ
continue;
}

var folder = web.EnsureFolderPath(folderName);
if (!knownFolders.TryGetValue(folderName, out var folder))
{
folder = web.EnsureFolderPath(folderName);

folder.EnsureProperties(p => p.UniqueId, p => p.ServerRelativeUrl);
parser.AddToken(new FileUniqueIdToken(web, folder.ServerRelativeUrl.Substring(web.ServerRelativeUrl.Length).TrimStart("/".ToCharArray()), folder.UniqueId));
parser.AddToken(new FileUniqueIdEncodedToken(web, folder.ServerRelativeUrl.Substring(web.ServerRelativeUrl.Length).TrimStart("/".ToCharArray()), folder.UniqueId));
knownFolders.Add(folderName, folder);
}

folder.EnsureProperties(p => p.UniqueId, p => p.ServerRelativeUrl);
parser.AddToken(new FileUniqueIdToken(web, folder.ServerRelativeUrl.Substring(web.ServerRelativeUrl.Length).TrimStart("/".ToCharArray()), folder.UniqueId));
parser.AddToken(new FileUniqueIdEncodedToken(web, folder.ServerRelativeUrl.Substring(web.ServerRelativeUrl.Length).TrimStart("/".ToCharArray()), folder.UniqueId));

var checkedOut = false;

var targetFile = folder.GetFile(template.Connector.GetFilenamePart(targetFileName));

var additionalRetrievals = new List<Expression<Func<File, object>>>()
{
f => f.UniqueId,
f => f.ServerRelativePath,
f => f.ListItemAllFields.Id,
f => f.Level
};
if (targetFile != null)
{
if (file.Overwrite)
Expand All @@ -112,7 +126,7 @@ public override TokenParser ProvisionObjects(Web web, ProvisioningTemplate templ
}
else
{
checkedOut = CheckOutIfNeeded(web, targetFile);
checkedOut = CheckOutIfNeeded(web, targetFile, additionalRetrievals.ToArray());
}
}
else
Expand All @@ -130,19 +144,21 @@ public override TokenParser ProvisionObjects(Web web, ProvisioningTemplate templ
}
}

checkedOut = CheckOutIfNeeded(web, targetFile);
checkedOut = CheckOutIfNeeded(web, targetFile, additionalRetrievals.ToArray());
}

if (targetFile != null)
{
// Add the fileuniqueid tokens
targetFile.EnsureProperties(p => p.UniqueId, p => p.ServerRelativePath);
// PERFORMANCE NOTE: next call not needed; already loaded in CheckOutIfNeeded via additionalRetrievals (save API calls)
// targetFile.EnsureProperties(p => p.UniqueId, p => p.ServerRelativePath);

// Add ListItemId token, given that a file can live outside of a library ensure this does not break provisioning
try
{
web.Context.Load(targetFile, p => p.ListItemAllFields.Id);
web.Context.ExecuteQueryRetry();
// PERFORMANCE NOTE: next 2 calls not needed; already loaded in CheckOutIfNeeded via additionalRetrievals (save API calls)
// web.Context.Load(targetFile, p => p.ListItemAllFields.Id);
// web.Context.ExecuteQueryRetry();
if (targetFile.ListItemAllFields.ServerObjectIsNull.HasValue
&& !targetFile.ListItemAllFields.ServerObjectIsNull.Value)
{
Expand Down Expand Up @@ -210,12 +226,18 @@ public override TokenParser ProvisionObjects(Web web, ProvisioningTemplate templ
{
case Model.FileLevel.Published:
{
targetFile.PublishFileToLevel(Microsoft.SharePoint.Client.FileLevel.Published);
if (targetFile.Level != Microsoft.SharePoint.Client.FileLevel.Published)
{
targetFile.PublishFileToLevel(Microsoft.SharePoint.Client.FileLevel.Published);
}
break;
}
case Model.FileLevel.Draft:
{
targetFile.PublishFileToLevel(Microsoft.SharePoint.Client.FileLevel.Draft);
if (targetFile.Level != Microsoft.SharePoint.Client.FileLevel.Draft)
{
targetFile.PublishFileToLevel(Microsoft.SharePoint.Client.FileLevel.Draft);
}
break;
}
default:
Expand Down Expand Up @@ -244,12 +266,19 @@ public override TokenParser ProvisionObjects(Web web, ProvisioningTemplate templ
return parser;
}

private static bool CheckOutIfNeeded(Web web, File targetFile)
private static bool CheckOutIfNeeded(Web web, File targetFile, params Expression<Func<File, object>>[] additionalRetrievals)
{
var checkedOut = false;
try
{
web.Context.Load(targetFile, f => f.CheckOutType, f => f.CheckedOutByUser, f => f.ListItemAllFields.ParentList.ForceCheckout);
var retrievals = new List<Expression<Func<File, object>>>
{
f => f.CheckOutType,
f => f.CheckedOutByUser,
f => f.ListItemAllFields.ParentList.ForceCheckout
};
retrievals.AddRange(additionalRetrievals);
web.Context.Load(targetFile, retrievals.ToArray());
web.Context.ExecuteQueryRetry();

if (targetFile.ListItemAllFields.ServerObjectIsNull.HasValue
Expand Down