Skip to content

Commit

Permalink
GetStorageAccountAsync is making a Storage Account management API cal…
Browse files Browse the repository at this point in the history
…l, which is restricted at 800 requests / 5 minutes. This is different from calls made to the storage account data place (which have a much higher threshold). https://learn.microsoft.com/en-us/azure/role-based-access-control/resource-provider-operations#microsoftstorage has list of APIs.
  • Loading branch information
srayan committed Feb 21, 2024
1 parent 55d80e9 commit f95d8cb
Showing 1 changed file with 16 additions and 2 deletions.
18 changes: 16 additions & 2 deletions migrationTool/ams/AssetMigrator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using Azure.Storage.Blobs;
using Microsoft.Extensions.Logging;
using Spectre.Console;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Threading.Channels;

Expand All @@ -16,6 +17,7 @@ internal class AssetMigrator : BaseMigrator
private readonly TransformFactory _transformFactory;
private readonly AssetOptions _options;
private readonly IMigrationTracker<BlobContainerClient, AssetMigrationResult> _tracker;
protected readonly TokenCredential _credentials;

public AssetMigrator(
GlobalOptions globalOptions,
Expand All @@ -30,6 +32,7 @@ public AssetMigrator(
_options = assetOptions;
_tracker = tracker;
_transformFactory = transformFactory;
_credentials = credential;
}

public override async Task MigrateAsync(CancellationToken cancellationToken)
Expand Down Expand Up @@ -83,11 +86,22 @@ public override async Task MigrateAsync(CancellationToken cancellationToken)
private async Task<AssetStats> MigrateAsync(MediaServicesAccountResource account, AsyncPageable<MediaAssetResource> assets, List<MediaAssetResource>? filteredList, ChannelWriter<double> writer, CancellationToken cancellationToken)
{
var stats = new AssetStats();

// Dictionary to restrict spawning of too many BlobServiceClients
var storageAccounts = new ConcurrentDictionary<string, BlobServiceClient>();

await MigrateInParallel(assets, filteredList, async (asset, cancellationToken) =>
{
var storage = await _resourceProvider.GetStorageAccountAsync(account, asset, cancellationToken);
var storageAccountName = asset.Data.StorageAccountName;
if (!storageAccounts.ContainsKey(storageAccountName))
{
BlobServiceClient newBlobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net"), _credentials);
storageAccounts.TryAdd(storageAccountName, newBlobServiceClient);
}

BlobServiceClient blobServiceClient = storageAccounts[storageAccountName];

var result = await MigrateAsync(account, storage, asset, cancellationToken);
var result = await MigrateAsync(account, blobServiceClient, asset, cancellationToken);
stats.Update(result);
await writer.WriteAsync(stats.Total, cancellationToken);

Expand Down

0 comments on commit f95d8cb

Please sign in to comment.