diff --git a/samples/Sample.CLI/Program.cs b/samples/Sample.CLI/Program.cs index 1f4989ba..fc53be79 100644 --- a/samples/Sample.CLI/Program.cs +++ b/samples/Sample.CLI/Program.cs @@ -66,9 +66,6 @@ class DownloadOptions [Option('k', "asset-key", Required = true, HelpText = "The name of the asset to download")] public string Key { get; set; } - [Option('e', "encoding", Required = true, HelpText = "The encoding of the asset to download (e.g UTF-8)")] - public string Encoding { get; set; } - [Option('i', "identity-pem-path", Required = false, HelpText = "The path to an identity PEM file to auth the download")] public string IdentityPEMFilePath { get; set; } @@ -117,7 +114,7 @@ await result Samples s = new(agent); Principal canisterId = Principal.FromText(options.CanisterId!); - await s.DownloadFileAsync(canisterId, options.Key, options.Encoding, options.FilePath); + await s.DownloadFileAsync(canisterId, options.Key, options.FilePath); }); } @@ -159,15 +156,14 @@ public Samples(IAgent agent) public async Task DownloadFileAsync( Principal canisterId, string key, - string encoding, string outputFilePath ) { - var client = new AssetCanisterApiClient(this.agent, canisterId); + AssetCanisterApiClient client = new(this.agent, canisterId); Console.WriteLine($"Downloading asset '{key}'..."); - GetResult result = await client.GetAsync(key, new List { encoding }); - File.WriteAllBytes(outputFilePath, result.Content); + byte[] assetBytes = await client.DownloadAssetAsync(key); + File.WriteAllBytes(outputFilePath, assetBytes); Console.WriteLine($"Downloaded asset '{key}' to {outputFilePath}"); } diff --git a/src/Agent/API.xml b/src/Agent/API.xml index 8f3cf9d3..653a75b4 100644 --- a/src/Agent/API.xml +++ b/src/Agent/API.xml @@ -4,45 +4,6 @@ EdjCase.ICP.Agent - - - An `IAgent` implementation using HTTP to make requests to the IC - - - - - The identity that will be used on each request unless overriden - This identity can be anonymous - - - - Optional. Identity to use for each request. If unspecified, will use anonymous identity - Optional. Bls crypto implementation to validate signatures. If unspecified, will use default implementation - Optional. Sets the http client to use, otherwise will use the default http client - - - Optional. Identity to use for each request. If unspecified, will use anonymous identity - Optional. Bls crypto implementation to validate signatures. If unspecified, will use default implementation - Url to the boundry node to connect to. Defaults to `https://ic0.app/` - - - - - - - - - - - - - - - - - - - The default http client to use with the built in `HttpClient` @@ -111,6 +72,45 @@ + + + An `IAgent` implementation using HTTP to make requests to the IC + + + + + The identity that will be used on each request unless overriden + This identity can be anonymous + + + + Optional. Identity to use for each request. If unspecified, will use anonymous identity + Optional. Bls crypto implementation to validate signatures. If unspecified, will use default implementation + Optional. Sets the http client to use, otherwise will use the default http client + + + Optional. Identity to use for each request. If unspecified, will use anonymous identity + Optional. Bls crypto implementation to validate signatures. If unspecified, will use default implementation + Url to the boundry node to connect to. Defaults to `https://ic0.app/` + + + + + + + + + + + + + + + + + + + An agent is used to communicate with the Internet Computer with certain protocols that @@ -1249,6 +1249,14 @@ Additional headers to be included in the request. The maximum age of the asset in seconds. + + + A helper method to download an asset from the asset canister in chunks. + + The key of the asset to download. + The maximum number of concurrent chunk downloads. + The downloaded asset content as a byte array. + Retrieves the API version for the asset canister diff --git a/src/Agent/Standards/AssetCanister/AssetCanisterApiClient.cs b/src/Agent/Standards/AssetCanister/AssetCanisterApiClient.cs index f12b1541..5c7d6aca 100644 --- a/src/Agent/Standards/AssetCanister/AssetCanisterApiClient.cs +++ b/src/Agent/Standards/AssetCanister/AssetCanisterApiClient.cs @@ -7,6 +7,9 @@ using System.IO; using System; using EdjCase.ICP.Agent.Standards.AssetCanister.Models; +using System.Linq; +using System.Threading; +using System.IO.Compression; namespace EdjCase.ICP.Agent.Standards.AssetCanister { @@ -23,7 +26,7 @@ public class AssetCanisterApiClient /// The maximum size of a file chunk /// It is set to just under 2MB. /// - public const int MAX_CHUNK_SIZE = MAX_INGRESS_MESSAGE_SIZE - 200; // Just under 2MB + public const int MAX_CHUNK_SIZE = MAX_INGRESS_MESSAGE_SIZE - 500; // Just under 2MB /// /// The IC agent @@ -136,7 +139,7 @@ public async Task UploadAssetChunkedAsync( break; } byte[] chunkBytes = bytesRead < buffer.Length - ? buffer[0..(bytesRead - 1)] + ? buffer[0..bytesRead] : buffer; CreateChunkResult result = await this.CreateChunkAsync(createBatchResult.BatchId, chunkBytes); chunkIds.Add(result.ChunkId); @@ -179,6 +182,100 @@ public async Task UploadAssetChunkedAsync( } } + /// + /// A helper method to download an asset from the asset canister in chunks. + /// + /// The key of the asset to download. + /// The maximum number of concurrent chunk downloads. + /// The downloaded asset content as a byte array. + public async Task DownloadAssetAsync(string key, int maxConcurrency = 10) + { + List acceptEncodings = new() { "identity", "gzip", "deflate", "br" }; + GetResult result = await this.GetAsync(key, acceptEncodings); + + if (!result.TotalLength.TryToUInt64(out ulong totalLength)) + { + throw new Exception("Total file length is too large: " + result.TotalLength); + } + if (totalLength == (ulong)result.Content.Length) + { + return result.Content; + } + int chunkCount = (int)Math.Ceiling((double)totalLength / result.Content.Length); + + // Create a list to store the chunk tasks + List> chunkTasks = new List>(); + + // Create a semaphore to limit the number of concurrent tasks + SemaphoreSlim semaphore = new(maxConcurrency); + + byte[]? sha256 = result.Sha256.GetValueOrDefault(); + // Download the rest of the chunks + // Skip the first chunk as we already have it + for (int i = 1; i < chunkCount; i++) + { + int chunkIndex = i; + chunkTasks.Add(Task.Run(async () => + { + await semaphore.WaitAsync(); + try + { + GetChunkResult chunkResult = await this.GetChunkAsync(key, result.ContentEncoding, UnboundedUInt.FromUInt64((ulong)chunkIndex), sha256); + return chunkResult.Content; + } + finally + { + semaphore.Release(); + } + })); + } + + // Wait for all chunk tasks to complete + await Task.WhenAll(chunkTasks); + + // Combine all the bytes into one byte[] + byte[] combinedBytes = result.Content.Concat(chunkTasks.SelectMany(t => t.Result)).ToArray(); + + switch (result.ContentEncoding) + { + case "identity": + case null: + case "": + break; + case "gzip": + using (var memoryStream = new MemoryStream(combinedBytes)) + using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) + using (var decompressedStream = new MemoryStream()) + { + gzipStream.CopyTo(decompressedStream); + combinedBytes = decompressedStream.ToArray(); + } + break; + case "deflate": + using (var memoryStream = new MemoryStream(combinedBytes)) + using (var deflateStream = new DeflateStream(memoryStream, CompressionMode.Decompress)) + using (var decompressedStream = new MemoryStream()) + { + deflateStream.CopyTo(decompressedStream); + combinedBytes = decompressedStream.ToArray(); + } + break; + case "br": + using (var memoryStream = new MemoryStream(combinedBytes)) + using (var brotliStream = new BrotliStream(memoryStream, CompressionMode.Decompress)) + using (var decompressedStream = new MemoryStream()) + { + brotliStream.CopyTo(decompressedStream); + combinedBytes = decompressedStream.ToArray(); + } + break; + default: + throw new NotImplementedException($"Content encoding {result.ContentEncoding} is not supported"); + } + + return combinedBytes; + } + /// /// Retrieves the API version for the asset canister ///