From 8de570f85683a6b07472fe88aa528fef2b7f7d28 Mon Sep 17 00:00:00 2001 From: KoenZomers Date: Tue, 3 Dec 2024 16:15:50 +0100 Subject: [PATCH 1/4] Added paramset for working with doclibs --- documentation/Get-PnPFolderItem.md | 2 +- src/Commands/Files/GetFolderItem.cs | 93 +++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 6 deletions(-) diff --git a/documentation/Get-PnPFolderItem.md b/documentation/Get-PnPFolderItem.md index c49e72714..b70ff238c 100644 --- a/documentation/Get-PnPFolderItem.md +++ b/documentation/Get-PnPFolderItem.md @@ -19,7 +19,7 @@ List files and/or subfolders in a folder Get-PnPFolderItem [-FolderSiteRelativeUrl ] [-ItemType ] [-ItemName ] [-Recursive] [-Verbose] [-Connection ] ``` -### Folder via pipebind +### Folder via folder pipebind ```powershell Get-PnPFolderItem [-Identity ] [-ItemType ] [-ItemName ] [-Recursive] [-Verbose] [-Connection ] ``` diff --git a/src/Commands/Files/GetFolderItem.cs b/src/Commands/Files/GetFolderItem.cs index 123b6364d..882a9ea15 100644 --- a/src/Commands/Files/GetFolderItem.cs +++ b/src/Commands/Files/GetFolderItem.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using System.Management.Automation; +using System.Xml.Linq; using Microsoft.SharePoint.Client; using PnP.Framework.Utilities; using PnP.PowerShell.Commands.Base.PipeBinds; @@ -14,7 +16,8 @@ namespace PnP.PowerShell.Commands.Files [OutputType(typeof(IEnumerable))] public class GetFolderItem : PnPWebCmdlet { - private const string ParameterSet_FOLDERSBYPIPE = "Folder via pipebind"; + private const string ParameterSet_FOLDERSBYPIPE = "Folder via folder pipebind"; + private const string ParameterSet_LISTSBYPIPE = "Folder via list pipebind"; private const string ParameterSet_FOLDERBYURL = "Folder via url"; [Parameter(Mandatory = false, Position = 0, ValueFromPipeline = true, ParameterSetName = ParameterSet_FOLDERBYURL)] @@ -23,6 +26,9 @@ public class GetFolderItem : PnPWebCmdlet [Parameter(Mandatory = false, Position = 0, ValueFromPipeline = true, ParameterSetName = ParameterSet_FOLDERSBYPIPE)] public FolderPipeBind Identity; + [Parameter(Mandatory = false, Position = 0, ValueFromPipeline = true, ParameterSetName = ParameterSet_LISTSBYPIPE)] + public ListPipeBind List; + [Parameter(Mandatory = false)] [ValidateSet("Folder", "File", "All")] public string ItemType = "All"; @@ -31,14 +37,28 @@ public class GetFolderItem : PnPWebCmdlet public string ItemName = string.Empty; [Alias("Recurse")] - [Parameter(Mandatory = false)] + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FOLDERBYURL)] + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FOLDERSBYPIPE)] public SwitchParameter Recursive; + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_LISTSBYPIPE)] + public string[] Includes; + protected override void ExecuteCmdlet() { CurrentWeb.EnsureProperty(w => w.ServerRelativeUrl); - var contents = GetContents(FolderSiteRelativeUrl); + IEnumerable contents = null; + if (ParameterSetName == ParameterSet_LISTSBYPIPE) + { + // Get the files and folders from the list, supporting large lists + contents = GetContentsFromDocumentLibrary(List.GetList(CurrentWeb)); + } + else + { + // Get the files and folders from the file system, not supporting large lists + contents = GetContentsByUrl(FolderSiteRelativeUrl); + } if (!string.IsNullOrEmpty(ItemName)) { @@ -67,7 +87,70 @@ protected override void ExecuteCmdlet() WriteObject(contents, true); } - private IEnumerable GetContents(string FolderSiteRelativeUrl) + private IEnumerable GetContentsFromDocumentLibrary(List documentLibrary) + { + var query = CamlQuery.CreateAllItemsQuery(); + var queryElement = XElement.Parse(query.ViewXml); + + var rowLimit = queryElement.Descendants("RowLimit").FirstOrDefault(); + if (rowLimit != null) + { + rowLimit.RemoveAll(); + } + else + { + rowLimit = new XElement("RowLimit"); + queryElement.Add(rowLimit); + } + + rowLimit.SetAttributeValue("Paged", "TRUE"); + rowLimit.SetValue(1000); + + query.ViewXml = queryElement.ToString(); + + List results = []; + + do + { + var listItems = documentLibrary.GetItems(query); + // Call ClientContext.Load() with and without retrievalExpressions to load FieldValues, otherwise no fields will be loaded (CSOM behavior) + ClientContext.Load(listItems); + ClientContext.Load(listItems, l => l.Include(l => l.FileSystemObjectType, l => l.File, l => l.Folder, l => l.Id, l => l.DisplayName, l => l["FileLeafRef"], l => l["FileRef"])); + //ClientContext.Load(listItems, l => l.Include(GetPropertyExpressions(new[] { "File.VersionExpirationReport" }))); + ClientContext.ExecuteQueryRetry(); + + foreach (var listItem in listItems) + { + if(listItem.FileSystemObjectType == FileSystemObjectType.File && (ItemType == "File" || ItemType == "All")) + { + results.Add(listItem.File); + } + if(listItem.FileSystemObjectType == FileSystemObjectType.Folder && (ItemType == "Folder" || ItemType == "All")) + { + results.Add(listItem.Folder); + } + } + results.AddRange(listItems); + + query.ListItemCollectionPosition = listItems.ListItemCollectionPosition; + } while (query.ListItemCollectionPosition != null); + + return results; + } + + protected Expression>[] GetPropertyExpressions(string[] fieldsToLoad) + { + var expressions = new List>>(); + foreach (var include in fieldsToLoad) + { + var exp = (Expression>)Utilities.DynamicExpression.ParseLambda(typeof(ListItem), typeof(object), include, null); + + expressions.Add(exp); + } + return expressions.ToArray(); + } + + private IEnumerable GetContentsByUrl(string FolderSiteRelativeUrl) { Folder targetFolder = null; if (string.IsNullOrEmpty(FolderSiteRelativeUrl) && ParameterSetName == ParameterSet_FOLDERSBYPIPE && Identity != null) @@ -127,7 +210,7 @@ private IEnumerable GetContents(string FolderSiteRelativeUrl) WriteVerbose($"Processing folder {relativeUrl}"); - var subFolderContents = GetContents(relativeUrl); + var subFolderContents = GetContentsByUrl(relativeUrl); folderContent = folderContent.Concat(subFolderContents); } } From fe2350a2061bd99504ecb3bfddbe7047e70100c7 Mon Sep 17 00:00:00 2001 From: KoenZomers Date: Tue, 3 Dec 2024 22:07:37 +0100 Subject: [PATCH 2/4] Finalized implementation of Get-PnPFolderItem --- documentation/Get-PnPFolderItem.md | 7 +++++ src/Commands/Files/GetFolderItem.cs | 41 +++++++++++------------------ 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/documentation/Get-PnPFolderItem.md b/documentation/Get-PnPFolderItem.md index b70ff238c..8b611bd7d 100644 --- a/documentation/Get-PnPFolderItem.md +++ b/documentation/Get-PnPFolderItem.md @@ -24,10 +24,17 @@ Get-PnPFolderItem [-FolderSiteRelativeUrl ] [-ItemType ] [-ItemN Get-PnPFolderItem [-Identity ] [-ItemType ] [-ItemName ] [-Recursive] [-Verbose] [-Connection ] ``` +### Folder via list pipebind +```powershell +Get-PnPFolderItem [-List ] [-ItemType ] [-ItemName ] [-Includes ] [-Verbose] [-Connection ] +``` + ## DESCRIPTION This cmdlet allows listing of all the content in a folder. It can be used to list all files and folders in a folder and optionally all its subfolders. +When working with a document library containing over 5,000 items in it, you will need to use the -List parameter to specify the document library in order to be able to retrieve the Files and Folders in it. It will always recursively retrieve all files and folders in the document library. It is still possible to use -ItemType to restrict the results to only files or folders. You can also use -Includes to fetch additional properties of the files and folders. Start the property name with "File." or "Folder." followed by the property name of the file or folder. For example, to include the file version history report, pass in -Includes "File.VersionExpirationReport". + Use [Get-PnPFileInFolder](Get-PnPFileInFolder.md) to retrieve only files and [Get-PnPFolderInFolder](Get-PnPFolderInFolder.md) to retrieve only folders allowing additional properties of the returned items to be requested. ## EXAMPLES diff --git a/src/Commands/Files/GetFolderItem.cs b/src/Commands/Files/GetFolderItem.cs index 882a9ea15..a263f7998 100644 --- a/src/Commands/Files/GetFolderItem.cs +++ b/src/Commands/Files/GetFolderItem.cs @@ -92,19 +92,10 @@ private IEnumerable GetContentsFromDocumentLibrary(List documentLi var query = CamlQuery.CreateAllItemsQuery(); var queryElement = XElement.Parse(query.ViewXml); - var rowLimit = queryElement.Descendants("RowLimit").FirstOrDefault(); - if (rowLimit != null) - { - rowLimit.RemoveAll(); - } - else - { - rowLimit = new XElement("RowLimit"); - queryElement.Add(rowLimit); - } - + var rowLimit = new XElement("RowLimit"); rowLimit.SetAttributeValue("Paged", "TRUE"); rowLimit.SetValue(1000); + queryElement.Add(rowLimit); query.ViewXml = queryElement.ToString(); @@ -115,8 +106,19 @@ private IEnumerable GetContentsFromDocumentLibrary(List documentLi var listItems = documentLibrary.GetItems(query); // Call ClientContext.Load() with and without retrievalExpressions to load FieldValues, otherwise no fields will be loaded (CSOM behavior) ClientContext.Load(listItems); - ClientContext.Load(listItems, l => l.Include(l => l.FileSystemObjectType, l => l.File, l => l.Folder, l => l.Id, l => l.DisplayName, l => l["FileLeafRef"], l => l["FileRef"])); - //ClientContext.Load(listItems, l => l.Include(GetPropertyExpressions(new[] { "File.VersionExpirationReport" }))); + ClientContext.Load(listItems, items => items.Include(item => item.FileSystemObjectType, + item => item.Id, + item => item.DisplayName, + item => item["FileLeafRef"], + item => item["FileRef"], + item => item.File, + item => item.Folder)); + + if(ParameterSpecified(nameof(Includes))) + { + var expressions = Includes.Select(i => (Expression>)Utilities.DynamicExpression.ParseLambda(typeof(ListItem), typeof(object), i, null)).ToArray(); + ClientContext.Load(listItems, items => items.Include(expressions)); + } ClientContext.ExecuteQueryRetry(); foreach (var listItem in listItems) @@ -130,6 +132,7 @@ private IEnumerable GetContentsFromDocumentLibrary(List documentLi results.Add(listItem.Folder); } } + results.AddRange(listItems); query.ListItemCollectionPosition = listItems.ListItemCollectionPosition; @@ -138,18 +141,6 @@ private IEnumerable GetContentsFromDocumentLibrary(List documentLi return results; } - protected Expression>[] GetPropertyExpressions(string[] fieldsToLoad) - { - var expressions = new List>>(); - foreach (var include in fieldsToLoad) - { - var exp = (Expression>)Utilities.DynamicExpression.ParseLambda(typeof(ListItem), typeof(object), include, null); - - expressions.Add(exp); - } - return expressions.ToArray(); - } - private IEnumerable GetContentsByUrl(string FolderSiteRelativeUrl) { Folder targetFolder = null; From c60bc7b0ab6ae0a5c5f191bd464467e57d71f7a6 Mon Sep 17 00:00:00 2001 From: KoenZomers Date: Tue, 3 Dec 2024 23:21:58 +0100 Subject: [PATCH 3/4] Added same logic to Get-PnPFileInFolder and Get-PnPFolderInFolder as well --- documentation/Get-PnPFileInFolder.md | 21 +++++++ documentation/Get-PnPFolderInFolder.md | 21 +++++++ documentation/Get-PnPFolderItem.md | 14 +++++ src/Commands/Files/GetFileInFolder.cs | 76 ++++++++++++++++++++++--- src/Commands/Files/GetFolderInFolder.cs | 75 +++++++++++++++++++++--- src/Commands/Files/GetFolderItem.cs | 2 - 6 files changed, 191 insertions(+), 18 deletions(-) diff --git a/documentation/Get-PnPFileInFolder.md b/documentation/Get-PnPFileInFolder.md index fd5c9babe..fb6833f4b 100644 --- a/documentation/Get-PnPFileInFolder.md +++ b/documentation/Get-PnPFileInFolder.md @@ -24,10 +24,17 @@ Get-PnPFileInFolder [-FolderSiteRelativeUrl ] [-ItemName ] [-Rec Get-PnPFileInFolder [-Identity ] [-ItemName ] [-Recurse] [-Includes ] [-ExcludeSystemFolders] [-Verbose] [-Connection ] ``` +### Folder via list pipebind +```powershell +Get-PnPFileInFolder [-List ] [-ItemType ] [-ItemName ] [-Includes ] [-Verbose] [-Connection ] +``` + ## DESCRIPTION This cmdlet allows listing of all the files in a folder. It can optionally also list all files in the underlying subfolders. +When working with a document library containing over 5,000 items in it, you will need to use the -List parameter to specify the document library in order to be able to retrieve the files in it. It will always recursively retrieve all files in the document library. You can also use -Includes to fetch additional properties of the files. + ## EXAMPLES ### EXAMPLE 1 @@ -165,6 +172,20 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -List +The document library to retrieve the files from. This parameter is required when working with document libraries containing over 5,000 items. + +```yaml +Type: ListPipeBind +Parameter Sets: Folder via list pipebind + +Required: False +Position: 0 +Default value: None +Accept pipeline input: True +Accept wildcard characters: False +``` + ### -Recurse A switch parameter to include files of all subfolders in the specified folder diff --git a/documentation/Get-PnPFolderInFolder.md b/documentation/Get-PnPFolderInFolder.md index 9880f5c6a..e55e15256 100644 --- a/documentation/Get-PnPFolderInFolder.md +++ b/documentation/Get-PnPFolderInFolder.md @@ -24,10 +24,17 @@ Get-PnPFolderInFolder [-FolderSiteRelativeUrl ] [-ItemName ] [-E Get-PnPFolderInFolder [-Identity ] [-ItemName ] [-ExcludeSystemFolders] [-Includes ] [-Recurse] [-Verbose] [-Connection ] ``` +### Folder via list pipebind +```powershell +Get-PnPFolderInFolder [-List ] [-ItemType ] [-ItemName ] [-Includes ] [-Verbose] [-Connection ] +``` + ## DESCRIPTION This cmdlet allows listing of all the subfolders of a folder. It can optionally also list all folders in the underlying subfolders. +When working with a document library containing over 5,000 items in it, you will need to use the -List parameter to specify the document library in order to be able to retrieve the Folders in it. It will always recursively retrieve all folders in the document library. You can also use -Includes to fetch additional properties of the folders. + ## EXAMPLES ### EXAMPLE 1 @@ -165,6 +172,20 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -List +The document library to retrieve the folders from. This parameter is required when working with document libraries containing over 5,000 items. + +```yaml +Type: ListPipeBind +Parameter Sets: Folder via list pipebind + +Required: False +Position: 0 +Default value: None +Accept pipeline input: True +Accept wildcard characters: False +``` + ### -Recurse A switch parameter to include folders of all subfolders in the specified folder diff --git a/documentation/Get-PnPFolderItem.md b/documentation/Get-PnPFolderItem.md index 8b611bd7d..ddf2390ab 100644 --- a/documentation/Get-PnPFolderItem.md +++ b/documentation/Get-PnPFolderItem.md @@ -161,6 +161,20 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -List +The document library to retrieve the files and folders from. This parameter is required when working with document libraries containing over 5,000 items. + +```yaml +Type: ListPipeBind +Parameter Sets: Folder via list pipebind + +Required: False +Position: 0 +Default value: None +Accept pipeline input: True +Accept wildcard characters: False +``` + ### -Recursive A switch parameter to include contents of all subfolders in the specified folder diff --git a/src/Commands/Files/GetFileInFolder.cs b/src/Commands/Files/GetFileInFolder.cs index f0e897f54..f71b5f940 100644 --- a/src/Commands/Files/GetFileInFolder.cs +++ b/src/Commands/Files/GetFileInFolder.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using System.Management.Automation; +using System.Xml.Linq; using Microsoft.SharePoint.Client; using PnP.Framework.Utilities; using PnP.PowerShell.Commands.Base.PipeBinds; @@ -15,6 +17,7 @@ namespace PnP.PowerShell.Commands.Files public class GetFileInFolder : PnPWebRetrievalsCmdlet { private const string ParameterSet_FOLDERSBYPIPE = "Folder via pipebind"; + private const string ParameterSet_LISTSBYPIPE = "Folder via list pipebind"; private const string ParameterSet_FOLDERBYURL = "Folder via url"; [Parameter(Mandatory = false, Position = 0, ValueFromPipeline = true, ParameterSetName = ParameterSet_FOLDERBYURL)] @@ -23,20 +26,36 @@ public class GetFileInFolder : PnPWebRetrievalsCmdlet [Parameter(Mandatory = false, Position = 0, ValueFromPipeline = true, ParameterSetName = ParameterSet_FOLDERSBYPIPE)] public FolderPipeBind Identity; + [Parameter(Mandatory = false, Position = 0, ValueFromPipeline = true, ParameterSetName = ParameterSet_LISTSBYPIPE)] + public ListPipeBind List; + [Parameter(Mandatory = false)] public string ItemName = string.Empty; - [Parameter(Mandatory = false)] - public SwitchParameter Recurse; + [Alias("Recurse")] + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FOLDERBYURL)] + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FOLDERSBYPIPE)] + public SwitchParameter Recursive; - [Parameter(Mandatory = false)] + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FOLDERBYURL)] + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FOLDERSBYPIPE)] public SwitchParameter ExcludeSystemFolders; protected override void ExecuteCmdlet() { CurrentWeb.EnsureProperty(w => w.ServerRelativeUrl); - var contents = GetContents(FolderSiteRelativeUrl); + IEnumerable contents = null; + if (ParameterSetName == ParameterSet_LISTSBYPIPE) + { + // Get the files from the list, supporting large lists + contents = GetContentsFromDocumentLibrary(List.GetList(CurrentWeb)); + } + else + { + // Get the files from the file system, not supporting large lists + contents = GetContentsByUrl(FolderSiteRelativeUrl); + } if (!string.IsNullOrEmpty(ItemName)) { @@ -46,7 +65,48 @@ protected override void ExecuteCmdlet() WriteObject(contents, true); } - private IEnumerable GetContents(string FolderSiteRelativeUrl) + private IEnumerable GetContentsFromDocumentLibrary(List documentLibrary) + { + var query = CamlQuery.CreateAllItemsQuery(); + var queryElement = XElement.Parse(query.ViewXml); + + var rowLimit = new XElement("RowLimit"); + rowLimit.SetAttributeValue("Paged", "TRUE"); + rowLimit.SetValue(1000); + queryElement.Add(rowLimit); + + query.ViewXml = queryElement.ToString(); + + List results = []; + + do + { + var listItems = documentLibrary.GetItems(query); + // Call ClientContext.Load() with and without retrievalExpressions to load FieldValues, otherwise no fields will be loaded (CSOM behavior) + ClientContext.Load(listItems); + ClientContext.Load(listItems, items => items.Include(item => item.FileSystemObjectType, + item => item.Id, + item => item.DisplayName, + item => item["FileLeafRef"], + item => item["FileRef"], + item => item.File)); + + if(ParameterSpecified(nameof(Includes))) + { + var expressions = Includes.Select(i => (Expression>)Utilities.DynamicExpression.ParseLambda(typeof(ListItem), typeof(object), $"File.{i}", null)).ToArray(); + ClientContext.Load(listItems, items => items.Include(expressions)); + } + ClientContext.ExecuteQueryRetry(); + + results.AddRange(listItems.Where(item => item.FileSystemObjectType == FileSystemObjectType.File).Select(item => item.File)); + + query.ListItemCollectionPosition = listItems.ListItemCollectionPosition; + } while (query.ListItemCollectionPosition != null); + + return results; + } + + private IEnumerable GetContentsByUrl(string FolderSiteRelativeUrl) { Folder targetFolder = null; if (string.IsNullOrEmpty(FolderSiteRelativeUrl) && ParameterSetName == ParameterSet_FOLDERSBYPIPE && Identity != null) @@ -81,7 +141,7 @@ private IEnumerable GetContents(string FolderSiteRelativeUrl) files = ClientContext.LoadQuery(targetFolder.Files.IncludeWithDefaultProperties(RetrievalExpressions)).OrderBy(f => f.Name); - if (Recurse) + if (Recursive) { if(ExcludeSystemFolders.ToBool()) { @@ -96,7 +156,7 @@ private IEnumerable GetContents(string FolderSiteRelativeUrl) IEnumerable folderContent = files; - if (Recurse && folders.Count() > 0) + if (Recursive && folders.Count() > 0) { foreach (var folder in folders) { @@ -104,7 +164,7 @@ private IEnumerable GetContents(string FolderSiteRelativeUrl) WriteVerbose($"Processing folder {relativeUrl}"); - var subFolderContents = GetContents(relativeUrl); + var subFolderContents = GetContentsByUrl(relativeUrl); folderContent = folderContent.Concat(subFolderContents); } } diff --git a/src/Commands/Files/GetFolderInFolder.cs b/src/Commands/Files/GetFolderInFolder.cs index 2ef743f91..f61a9a4ee 100644 --- a/src/Commands/Files/GetFolderInFolder.cs +++ b/src/Commands/Files/GetFolderInFolder.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Linq.Expressions; using System.Management.Automation; +using System.Xml.Linq; using Microsoft.SharePoint.Client; using PnP.Framework.Utilities; using PnP.PowerShell.Commands.Base.PipeBinds; @@ -15,6 +16,7 @@ namespace PnP.PowerShell.Commands.Files public class GetFolderInFolder : PnPWebRetrievalsCmdlet { private const string ParameterSet_FOLDERSBYPIPE = "Folder via pipebind"; + private const string ParameterSet_LISTSBYPIPE = "Folder via list pipebind"; private const string ParameterSet_FOLDERBYURL = "Folder via url"; [Parameter(Mandatory = false, ValueFromPipeline = true, Position = 0, ParameterSetName = ParameterSet_FOLDERBYURL)] @@ -22,14 +24,20 @@ public class GetFolderInFolder : PnPWebRetrievalsCmdlet [Parameter(Mandatory = false, ValueFromPipeline = true, Position = 0, ParameterSetName = ParameterSet_FOLDERSBYPIPE)] public FolderPipeBind Identity; + + [Parameter(Mandatory = false, Position = 0, ValueFromPipeline = true, ParameterSetName = ParameterSet_LISTSBYPIPE)] + public ListPipeBind List; [Parameter(Mandatory = false)] public string ItemName = string.Empty; - [Parameter(Mandatory = false)] - public SwitchParameter Recurse; + [Alias("Recurse")] + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FOLDERBYURL)] + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FOLDERSBYPIPE)] + public SwitchParameter Recursive; - [Parameter(Mandatory = false)] + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FOLDERBYURL)] + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FOLDERSBYPIPE)] public SwitchParameter ExcludeSystemFolders; protected override void ExecuteCmdlet() @@ -38,10 +46,20 @@ protected override void ExecuteCmdlet() if(ExcludeSystemFolders.ToBool()) { - DefaultRetrievalExpressions = new Expression>[] { f => f.ListItemAllFields }; + DefaultRetrievalExpressions = [f => f.ListItemAllFields]; } - var contents = GetContents(FolderSiteRelativeUrl); + IEnumerable contents = null; + if (ParameterSetName == ParameterSet_LISTSBYPIPE) + { + // Get the folders from the list, supporting large lists + contents = GetContentsFromDocumentLibrary(List.GetList(CurrentWeb)); + } + else + { + // Get the folders from the file system, not supporting large lists + contents = GetContentsByUrl(FolderSiteRelativeUrl); + } if (!string.IsNullOrEmpty(ItemName)) { @@ -51,7 +69,48 @@ protected override void ExecuteCmdlet() WriteObject(contents, true); } - private IEnumerable GetContents(string FolderSiteRelativeUrl) + private IEnumerable GetContentsFromDocumentLibrary(List documentLibrary) + { + var query = CamlQuery.CreateAllItemsQuery(); + var queryElement = XElement.Parse(query.ViewXml); + + var rowLimit = new XElement("RowLimit"); + rowLimit.SetAttributeValue("Paged", "TRUE"); + rowLimit.SetValue(1000); + queryElement.Add(rowLimit); + + query.ViewXml = queryElement.ToString(); + + List results = []; + + do + { + var listItems = documentLibrary.GetItems(query); + // Call ClientContext.Load() with and without retrievalExpressions to load FieldValues, otherwise no fields will be loaded (CSOM behavior) + ClientContext.Load(listItems); + ClientContext.Load(listItems, items => items.Include(item => item.FileSystemObjectType, + item => item.Id, + item => item.DisplayName, + item => item["FileLeafRef"], + item => item["FileRef"], + item => item.Folder)); + + if(ParameterSpecified(nameof(Includes))) + { + var expressions = Includes.Select(i => (Expression>)Utilities.DynamicExpression.ParseLambda(typeof(ListItem), typeof(object), $"Folder.{i}", null)).ToArray(); + ClientContext.Load(listItems, items => items.Include(expressions)); + } + ClientContext.ExecuteQueryRetry(); + + results.AddRange(listItems.Where(item => item.FileSystemObjectType == FileSystemObjectType.Folder).Select(item => item.Folder)); + + query.ListItemCollectionPosition = listItems.ListItemCollectionPosition; + } while (query.ListItemCollectionPosition != null); + + return results; + } + + private IEnumerable GetContentsByUrl(string FolderSiteRelativeUrl) { Folder targetFolder = null; if (string.IsNullOrEmpty(FolderSiteRelativeUrl) && ParameterSetName == ParameterSet_FOLDERSBYPIPE && Identity != null) @@ -95,7 +154,7 @@ private IEnumerable GetContents(string FolderSiteRelativeUrl) IEnumerable folderContent = folders; - if (Recurse && folders.Count() > 0) + if (Recursive && folders.Count() > 0) { foreach (var folder in folders) { @@ -103,7 +162,7 @@ private IEnumerable GetContents(string FolderSiteRelativeUrl) WriteVerbose($"Processing folder {relativeUrl}"); - var subFolderContents = GetContents(relativeUrl); + var subFolderContents = GetContentsByUrl(relativeUrl); folderContent = folderContent.Concat(subFolderContents); } } diff --git a/src/Commands/Files/GetFolderItem.cs b/src/Commands/Files/GetFolderItem.cs index a263f7998..3fdb466d3 100644 --- a/src/Commands/Files/GetFolderItem.cs +++ b/src/Commands/Files/GetFolderItem.cs @@ -133,8 +133,6 @@ private IEnumerable GetContentsFromDocumentLibrary(List documentLi } } - results.AddRange(listItems); - query.ListItemCollectionPosition = listItems.ListItemCollectionPosition; } while (query.ListItemCollectionPosition != null); From e60c8bbaf6c4ce661500ad70114ac66b7f62a8bb Mon Sep 17 00:00:00 2001 From: KoenZomers Date: Tue, 3 Dec 2024 23:43:48 +0100 Subject: [PATCH 4/4] Added changelog entry --- CHANGELOG.md | 1 + src/Commands/Base/GetManagedAppId.cs | 9 ++------- src/Commands/Base/RemoveManagedAppId.cs | 9 ++++----- src/Commands/Base/SetManagedAppId.cs | 6 +----- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30f301f4f..66ac8d153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added `-Files` parameter for `Send-PnPMail` cmdlet to allow files to be downloaded from SharePoint and then sent as attachments. - Added `-Force` parameter to `Set-PnPPropertyBagValue` cmdlet to toggle NoScript status of the site. - Added `-Batch` parameter to `Invoke-PnPGraphMethod` cmdlet to allow adding request in a batch. +- Added `-List` parameter to `Get-PnPFolderItem`, `Get-PnPFileInFolder` and `Get-PnPFolderInFolder` which allows them to work with a document library containing more than 5,000 items [#4611](https://github.com/pnp/powershell/pull/4611) ### Changed diff --git a/src/Commands/Base/GetManagedAppId.cs b/src/Commands/Base/GetManagedAppId.cs index 41dc371d2..1e29bf453 100644 --- a/src/Commands/Base/GetManagedAppId.cs +++ b/src/Commands/Base/GetManagedAppId.cs @@ -1,10 +1,5 @@ using System; using System.Management.Automation; -using System.Net; -using Microsoft.SharePoint.Client; -using PnP.Framework.Utilities; - -using PnP.PowerShell.Commands.Enums; namespace PnP.PowerShell.Commands.Base { @@ -12,7 +7,7 @@ namespace PnP.PowerShell.Commands.Base [OutputType(typeof(PSCredential))] public class GetManagedAppId : PSCmdlet { - [Parameter(Mandatory = true)] + [Parameter(Mandatory = true, Position = 0)] public string Url; protected override void ProcessRecord() @@ -25,7 +20,7 @@ protected override void ProcessRecord() } else { - WriteError(new ErrorRecord(new System.Exception("AppId not found"), "APPIDNOTFOUND", ErrorCategory.AuthenticationError, this)); + WriteError(new ErrorRecord(new Exception("AppId not found"), "APPIDNOTFOUND", ErrorCategory.AuthenticationError, this)); } } } diff --git a/src/Commands/Base/RemoveManagedAppId.cs b/src/Commands/Base/RemoveManagedAppId.cs index 493cd3676..3b67a8048 100644 --- a/src/Commands/Base/RemoveManagedAppId.cs +++ b/src/Commands/Base/RemoveManagedAppId.cs @@ -1,14 +1,13 @@ using System; using System.Management.Automation; - namespace PnP.PowerShell.Commands.Base { [Cmdlet(VerbsCommon.Remove, "PnPManagedAppId")] [OutputType(typeof(void))] public class RemoveManagedAppId : PSCmdlet { - [Parameter(Mandatory = true)] + [Parameter(Mandatory = true, Position = 0)] public string Url; [Parameter(Mandatory = false)] @@ -17,20 +16,20 @@ public class RemoveManagedAppId : PSCmdlet protected override void ProcessRecord() { Uri uri = new Uri(Url); - var appId = Utilities.CredentialManager.GetAppId((uri.ToString())); + var appId = Utilities.CredentialManager.GetAppId(uri.ToString()); if (appId != null) { if (Force || ShouldContinue($"Remove App Id: {Url}?", Properties.Resources.Confirm)) { if (!Utilities.CredentialManager.RemoveAppid(uri.ToString())) { - WriteError(new ErrorRecord(new System.Exception($"AppId for {Url} not removed"), "APPIDNOTREMOVED", ErrorCategory.WriteError, Url)); + WriteError(new ErrorRecord(new Exception($"AppId for {Url} not removed"), "APPIDNOTREMOVED", ErrorCategory.WriteError, Url)); } } } else { - WriteError(new ErrorRecord(new System.Exception($"AppId not found for {Url}"), "APPIDNOTFOUND", ErrorCategory.ObjectNotFound, Url)); + WriteError(new ErrorRecord(new Exception($"AppId not found for {Url}"), "APPIDNOTFOUND", ErrorCategory.ObjectNotFound, Url)); } } } diff --git a/src/Commands/Base/SetManagedAppId.cs b/src/Commands/Base/SetManagedAppId.cs index 8eba6a318..9cfde567a 100644 --- a/src/Commands/Base/SetManagedAppId.cs +++ b/src/Commands/Base/SetManagedAppId.cs @@ -1,9 +1,5 @@ using System; using System.Management.Automation; -using System.Security; -using PnP.Framework.Utilities; - -using PnP.PowerShell.Commands.Enums; namespace PnP.PowerShell.Commands.Base { @@ -11,7 +7,7 @@ namespace PnP.PowerShell.Commands.Base [OutputType(typeof(void))] public class SetManagedAppId : PSCmdlet { - [Parameter(Mandatory = true)] + [Parameter(Mandatory = true, Position = 0)] public string Url; [Parameter(Mandatory = true)]