Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Lease index blob between read and write #6

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Lease index blob between read and write #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 20 additions & 8 deletions src/QMS.Storage.AzureStorage/AzureStorageService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public AzureStorageService(IOptions<AzureStorageConfig> config)
/// <summary>
/// Store the given fileData in the blobstorage and return a blob identifying the results
/// </summary>
public async Task<ICloudBlob> StoreFileAsync(byte[] fileData, string contentType, string? fileName = null, string? containerName = null)
public async Task<ICloudBlob> StoreFileAsync(byte[] fileData, string contentType, string? fileName = null, string? containerName = null, string? leaseId = null)
{
var blobContainer = await GetBlobContainer(containerName).ConfigureAwait(false);
if (string.IsNullOrEmpty(fileName))
Expand All @@ -40,7 +40,10 @@ public async Task<ICloudBlob> StoreFileAsync(byte[] fileData, string contentType

using (var stream = new MemoryStream(fileData, writable: false))
{
await blockBlob.UploadFromStreamAsync(stream).ConfigureAwait(false);
if(leaseId != null)
await blockBlob.UploadFromStreamAsync(stream, new AccessCondition { LeaseId = leaseId }, null, null).ConfigureAwait(false);
else
await blockBlob.UploadFromStreamAsync(stream).ConfigureAwait(false);
}

return blockBlob;
Expand Down Expand Up @@ -107,15 +110,19 @@ public async Task<ICloudBlob> GetFileReference(string blobStoreId, string? conta
throw new InvalidDataException();
}


return blobReference;
}

public async Task<T?> ReadFileAsJson<T>(string fileName) where T : class
public async Task<(T?, string? leaseId)> ReadFileAsJson<T>(string fileName, TimeSpan? leaseTime = null) where T : class
{
string? leaseId = null;
ICloudBlob? blob = null;
try
{
var blob = await GetFileReference(fileName).ConfigureAwait(false);
blob = await GetFileReference(fileName).ConfigureAwait(false);

if (leaseTime.HasValue)
leaseId = await blob.AcquireLeaseAsync(leaseTime);

using (var stream = new MemoryStream())
{
Expand All @@ -127,22 +134,27 @@ public async Task<ICloudBlob> GetFileReference(string blobStoreId, string? conta

var cmsItem = JsonSerializer.Deserialize<T>(json);

return cmsItem;
return (cmsItem, leaseId);
}
}
catch (FileNotFoundException)
{
return default;
}
finally
{
if (leaseId != null && blob != null)
await blob.ReleaseLeaseAsync(new AccessCondition { LeaseId = leaseId });
}
}

public Task WriteFileAsJson<T>(T item, string fileName)
public Task WriteFileAsJson<T>(T item, string fileName, string? leaseId = null)
{
var json = JsonSerializer.Serialize(item);

byte[] fileData = Encoding.ASCII.GetBytes(json);

return StoreFileAsync(fileData, "application/json", fileName);
return StoreFileAsync(fileData, "application/json", fileName, leaseId: leaseId);
}

public async Task<IEnumerable<IListBlobItem>> GetFilesFromDirectory(string path, string? containerName = null)
Expand Down
97 changes: 67 additions & 30 deletions src/QMS.Storage.AzureStorage/CmsItemStorageService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Microsoft.Azure.Storage.Blob;
using Microsoft.Azure.Storage;
using Microsoft.Azure.Storage.Blob;
using Microsoft.Extensions.Options;
using QMS.Models;
using QMS.Storage.Interfaces;
Expand All @@ -16,6 +17,7 @@ public class CmsItemStorageService : IReadCmsItem, IWriteCmsItem
{
private readonly AzureStorageService azureStorageService;
private readonly CmsConfiguration cmsConfiguration;
private const string INDEX_FILE_NAME = "_index";

public bool CanSort => true;

Expand All @@ -28,10 +30,10 @@ public CmsItemStorageService(AzureStorageService azureStorageService, IOptions<C
public async Task<(IReadOnlyList<CmsItem> results, int total)> List(string cmsType, string? sortField, string? sortOrder, int pageSize = 20, int pageIndex = 0)
{
//Get index file
var indexFileName = GenerateFileName(cmsType, "_index", null);
var indexFileName = GenerateFileName(cmsType, INDEX_FILE_NAME, null);

//Get current index file
var indexFile = await azureStorageService.ReadFileAsJson<List<CmsItem>>(indexFileName).ConfigureAwait(false);
var (indexFile, _) = await azureStorageService.ReadFileAsJson<List<CmsItem>>(indexFileName).ConfigureAwait(false);
indexFile = indexFile ?? new List<CmsItem>();

var returnItems = indexFile.AsQueryable();
Expand Down Expand Up @@ -68,11 +70,13 @@ public CmsItemStorageService(AzureStorageService azureStorageService, IOptions<C
//return (result, total);
}

public Task<CmsItem?> Read(string cmsType, Guid id, string? lang)
public async Task<CmsItem?> Read(string cmsType, Guid id, string? lang)
{
var fileName = GenerateFileName(cmsType, id, lang);

return azureStorageService.ReadFileAsJson<CmsItem>(fileName);
var (indexFile, _) = await azureStorageService.ReadFileAsJson<CmsItem>(fileName);

return indexFile;
}

public async Task Write(CmsItem item, string cmsType, Guid id, string? lang)
Expand All @@ -81,34 +85,49 @@ public async Task Write(CmsItem item, string cmsType, Guid id, string? lang)
await azureStorageService.WriteFileAsJson(item, fileName);

//Write index file for paging and sorting
var indexFileName = GenerateFileName(cmsType, "_index", lang);
var indexFileName = GenerateFileName(cmsType, INDEX_FILE_NAME, lang);
var typeInfo = cmsConfiguration.Entities.Where(x => x.Key == cmsType).FirstOrDefault();
string? leaseId = null;

if (typeInfo == null)
return;

//Get current index file
var indexFile = await azureStorageService.ReadFileAsJson<List<CmsItem>>(indexFileName).ConfigureAwait(false);
indexFile = indexFile ?? new List<CmsItem>();
try
{
//Get current index file
var (indexFile, newLeaseId) = await azureStorageService.ReadFileAsJson<List<CmsItem>>(indexFileName, TimeSpan.FromSeconds(20)).ConfigureAwait(false);
indexFile = indexFile ?? new List<CmsItem>();
leaseId = newLeaseId;

//Remove existing item
indexFile.Remove(indexFile.Where(x => x.Id == item.Id).FirstOrDefault());
//Remove existing item
indexFile.Remove(indexFile.Where(x => x.Id == item.Id).FirstOrDefault());

var indexItem = new CmsItem {
Id = id,
CmsType = cmsType,
LastModifiedDate = item.LastModifiedDate
};
var indexItem = new CmsItem
{
Id = id,
CmsType = cmsType,
LastModifiedDate = item.LastModifiedDate
};

foreach (var prop in typeInfo.ListViewProperties)
{
var value = item.AdditionalProperties[prop.Key];
indexItem.AdditionalProperties[prop.Key] = value;
}
foreach (var prop in typeInfo.ListViewProperties)
{
var value = item.AdditionalProperties[prop.Key];
indexItem.AdditionalProperties[prop.Key] = value;
}

indexFile.Add(indexItem);
indexFile.Add(indexItem);

await azureStorageService.WriteFileAsJson(indexFile, indexFileName);
await azureStorageService.WriteFileAsJson(indexFile, indexFileName, leaseId);
}
finally
{
//Release lease
if (leaseId != null)
{
var file = await azureStorageService.GetFileReference(indexFileName);
await file.ReleaseLeaseAsync(new AccessCondition { LeaseId = leaseId });
}
}
}


Expand All @@ -130,14 +149,32 @@ public async Task Delete(string cmsType, Guid id, string? lang)
}

//Write index file for paging and sorting
var indexFileName = GenerateFileName(cmsType, "_index", lang);
//Get current index file
var indexFile = await azureStorageService.ReadFileAsJson<List<CmsItem>>(indexFileName).ConfigureAwait(false);
indexFile = indexFile ?? new List<CmsItem>();
var indexFileName = GenerateFileName(cmsType, INDEX_FILE_NAME, lang);
string? leaseId = null;

//Remove existing item
indexFile.Remove(indexFile.Where(x => x.Id == id).FirstOrDefault());
await azureStorageService.WriteFileAsJson(indexFile, indexFileName);
try
{
//Get current index file
var (indexFile, newLeaseId) = await azureStorageService.ReadFileAsJson<List<CmsItem>>(indexFileName, TimeSpan.FromSeconds(20)).ConfigureAwait(false);
indexFile = indexFile ?? new List<CmsItem>();
leaseId = newLeaseId;

//Remove existing item
indexFile.Remove(indexFile.Where(x => x.Id == id).FirstOrDefault());
await azureStorageService.WriteFileAsJson(indexFile, indexFileName);
}
finally
{
//Release lease
if (leaseId != null)
{
var file = await azureStorageService.GetFileReference(indexFileName);
await file.ReleaseLeaseAsync(new AccessCondition
{
LeaseId = leaseId
});
}
}
}

private static string GenerateFileName(string cmsType, Guid id, string? lang)
Expand Down