Skip to content

Commit

Permalink
feat: added a clearOnly and a snapshot backupDB cli command switch
Browse files Browse the repository at this point in the history
  • Loading branch information
@ericbrunner
ericbrunner committed Dec 16, 2024
1 parent bc1d7b9 commit 9a1c984
Show file tree
Hide file tree
Showing 11 changed files with 139 additions and 41 deletions.
13 changes: 6 additions & 7 deletions ...onsumerApi.Tests.Performance.SnapshotCreator.Tests/Features/Create/CreateSnapshotTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,15 @@ public class CreateSnapshotTests : SnapshotCreatorTestsBase
private readonly IMediator _mediator;
private readonly IOutputHelper _outputHelper;
private readonly CreateSnapshot.CommandHandler _sut;
private readonly IDatabaseRestoreHelper _databaseRestoreHelper;

public CreateSnapshotTests()
{
var logger = A.Fake<ILogger<CreateSnapshot.CommandHandler>>();
_poolConfigurationJsonReader = A.Fake<IPoolConfigurationJsonReader>();
_mediator = A.Fake<IMediator>();
_outputHelper = A.Fake<IOutputHelper>();
_databaseRestoreHelper = A.Fake<IDatabaseRestoreHelper>();
_sut = new CreateSnapshot.CommandHandler(logger, _poolConfigurationJsonReader, _mediator, _outputHelper, _databaseRestoreHelper);
var databaseRestoreHelper = A.Fake<IDatabaseRestoreHelper>();
_sut = new CreateSnapshot.CommandHandler(logger, _poolConfigurationJsonReader, _mediator, _outputHelper, databaseRestoreHelper);
}

[Theory]
Expand All @@ -37,7 +36,7 @@ public async Task Handle_ShouldReturnSuccessStatusMessage_WhenPoolConfigurationI
{
// Arrange
var fullFilePath = GetFullFilePath(poolConfigJsonFilename);
var command = new CreateSnapshot.Command("http://baseaddress", "clientId", "clientSecret", fullFilePath, ClearDatabase: false);
var command = new CreateSnapshot.Command("http://baseaddress", "clientId", "clientSecret", fullFilePath, ClearDatabase: false, BackupDatabase: false);

Check failure on line 39 in Applications/ConsumerApi/test/ConsumerApi.Tests.Performance.SnapshotCreator.Tests/Features/Create/CreateSnapshotTests.cs

View workflow job for this annotation

GitHub Actions / Run Unit Tests 

There is no argument given that corresponds to the required parameter 'ClearOnly' of 'CreateSnapshot.Command.Command(string, string, string, string, bool, bool, bool)'

Check failure on line 39 in Applications/ConsumerApi/test/ConsumerApi.Tests.Performance.SnapshotCreator.Tests/Features/Create/CreateSnapshotTests.cs

View workflow job for this annotation

GitHub Actions / Run Unit Tests 

There is no argument given that corresponds to the required parameter 'ClearOnly' of 'CreateSnapshot.Command.Command(string, string, string, string, bool, bool, bool)'

A.CallTo(() => _poolConfigurationJsonReader.Read(command.JsonFilePath)).Returns(
new PerformanceTestConfiguration(
Expand Down Expand Up @@ -83,7 +82,7 @@ public async Task Handle_ShouldReturnFailureStatusMessage_WhenPoolConfigurationI
{
// Arrange
var fullFilePath = GetFullFilePath(poolConfigJsonFilename);
var command = new CreateSnapshot.Command("http://baseaddress", "clientId", "clientSecret", fullFilePath, ClearDatabase: false);
var command = new CreateSnapshot.Command("http://baseaddress", "clientId", "clientSecret", fullFilePath, ClearDatabase: false, BackupDatabase: false);

Check failure on line 85 in Applications/ConsumerApi/test/ConsumerApi.Tests.Performance.SnapshotCreator.Tests/Features/Create/CreateSnapshotTests.cs

View workflow job for this annotation

GitHub Actions / Run Unit Tests 

There is no argument given that corresponds to the required parameter 'ClearOnly' of 'CreateSnapshot.Command.Command(string, string, string, string, bool, bool, bool)'

Check failure on line 85 in Applications/ConsumerApi/test/ConsumerApi.Tests.Performance.SnapshotCreator.Tests/Features/Create/CreateSnapshotTests.cs

View workflow job for this annotation

GitHub Actions / Run Unit Tests 

There is no argument given that corresponds to the required parameter 'ClearOnly' of 'CreateSnapshot.Command.Command(string, string, string, string, bool, bool, bool)'

A.CallTo(() => _poolConfigurationJsonReader.Read(command.JsonFilePath)).Returns(null as PerformanceTestConfiguration);

Expand All @@ -103,7 +102,7 @@ public async Task Handle_ShouldReturnFailureStatusMessage_WhenPoolConfigurationI
public async Task Handle_ShouldReturnFailureStatusMessage_WhenExceptionIsThrown()
{
// Arrange
var command = new CreateSnapshot.Command("http://baseaddress", "clientId", "clientSecret", GetFullFilePath("pool-config.test.json"), ClearDatabase: false);
var command = new CreateSnapshot.Command("http://baseaddress", "clientId", "clientSecret", GetFullFilePath("pool-config.test.json"), ClearDatabase: false, BackupDatabase: false);

Check failure on line 105 in Applications/ConsumerApi/test/ConsumerApi.Tests.Performance.SnapshotCreator.Tests/Features/Create/CreateSnapshotTests.cs

View workflow job for this annotation

GitHub Actions / Run Unit Tests 

There is no argument given that corresponds to the required parameter 'ClearOnly' of 'CreateSnapshot.Command.Command(string, string, string, string, bool, bool, bool)'

Check failure on line 105 in Applications/ConsumerApi/test/ConsumerApi.Tests.Performance.SnapshotCreator.Tests/Features/Create/CreateSnapshotTests.cs

View workflow job for this annotation

GitHub Actions / Run Unit Tests 

There is no argument given that corresponds to the required parameter 'ClearOnly' of 'CreateSnapshot.Command.Command(string, string, string, string, bool, bool, bool)'
var expectedException = new Exception("some exception");
A.CallTo(() => _poolConfigurationJsonReader.Read(command.JsonFilePath)).ThrowsAsync(expectedException);

Expand All @@ -124,7 +123,7 @@ public async Task Handle_ShouldReturnFailureStatusMessage_WhenPoolConfigurationF
{
// Arrange
var fullFilePath = GetFullFilePath("not-existing-pool-config.test.json");
var command = new CreateSnapshot.Command("http://baseaddress", "clientId", "clientSecret", fullFilePath, ClearDatabase: false);
var command = new CreateSnapshot.Command("http://baseaddress", "clientId", "clientSecret", fullFilePath, ClearDatabase: false, BackupDatabase: false);

Check failure on line 126 in Applications/ConsumerApi/test/ConsumerApi.Tests.Performance.SnapshotCreator.Tests/Features/Create/CreateSnapshotTests.cs

View workflow job for this annotation

GitHub Actions / Run Unit Tests 

There is no argument given that corresponds to the required parameter 'ClearOnly' of 'CreateSnapshot.Command.Command(string, string, string, string, bool, bool, bool)'

Check failure on line 126 in Applications/ConsumerApi/test/ConsumerApi.Tests.Performance.SnapshotCreator.Tests/Features/Create/CreateSnapshotTests.cs

View workflow job for this annotation

GitHub Actions / Run Unit Tests 

There is no argument given that corresponds to the required parameter 'ClearOnly' of 'CreateSnapshot.Command.Command(string, string, string, string, bool, bool, bool)'

// Act
var result = await _sut.Handle(command, CancellationToken.None);
Expand Down
31 changes: 29 additions & 2 deletions ...ConsumerApi.Tests.Performance/tools/snapshot-creator-v2/Features/Create/CreateSnapshot.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ public record Command(
string ClientId,
string ClientSecret,
string JsonFilePath,
bool ClearDatabase) : IRequest<StatusMessage>;
bool ClearDatabase,
bool BackupDatabase,
bool ClearOnly) : IRequest<StatusMessage>;

public class CommandHandler(
ILogger<CommandHandler> logger,
Expand All @@ -32,6 +34,19 @@ public async Task<StatusMessage> Handle(Command request, CancellationToken cance
{
try
{
if (request.ClearOnly)
{
var result = await databaseRestoreHelper.RestoreCleanDatabase();

if (!result.Status)
{
return result;
}

logger.LogInformation("Restore clean-db completed: {Message}", result.Message);
return new StatusMessage(true, CLEAN_DB_SUCCEED_MESSAGE);
}

logger.LogInformation("Creating pool configuration with relationships and messages ...");

if (request.ClearDatabase)
Expand All @@ -43,7 +58,7 @@ public async Task<StatusMessage> Handle(Command request, CancellationToken cance
return result;
}

logger.LogInformation("RestoreCleanDatabase: {Message}", result.Message);
logger.LogInformation("Restore clean-db completed: {Message}", result.Message);
}


Expand Down Expand Up @@ -122,6 +137,18 @@ public async Task<StatusMessage> Handle(Command request, CancellationToken cance
logger.LogInformation("Messages created in {ElapsedTime}", stopwatch.Elapsed);

logger.LogInformation("Pool configuration with relationships and messages created in {ElapsedTime}", totalRunTime);

if (request.BackupDatabase)
{
var result = await databaseRestoreHelper.BackupDatabase(OutputDirName);

if (!result.Status)
{
return result;
}

logger.LogInformation("Backup completed: {Message}", result.Message);
}
}
catch (Exception e)
{
Expand Down
98 changes: 75 additions & 23 deletions ...sts.Performance/tools/snapshot-creator-v2/Features/Create/Helper/DatabaseRestoreHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ public class DatabaseRestoreHelper(ILogger<DatabaseRestoreHelper> logger, IProce
private const string DB_NAME = "enmeshed";
private const string CONTAINER_NAME = "tmp-postgres-container";
private const string CLEAN_DB_NAME = "clean-db.rg";
private const string SNAPHOT_DB_NAME = "snapshot-db.rg";

private static bool QueryUserConsent()
{
Expand All @@ -25,7 +26,7 @@ private static bool QueryUserConsent()
return input == 'y';
}

public async Task<DatabaseRestoreResult> RestoreCleanDatabase()
public async Task<DatabaseResult> RestoreCleanDatabase()
{
try
{
Expand All @@ -43,8 +44,7 @@ public async Task<DatabaseRestoreResult> RestoreCleanDatabase()
switch (checkForOpenConnectionsResult.Status)
{
case false when checkForOpenConnectionsResult.IsError:
//Note; Error shouldn't stop the snapshot creation process, therefore, return true
return new DatabaseRestoreResult(true, checkForOpenConnectionsResult.Message);
return new DatabaseResult(false, checkForOpenConnectionsResult.Message);
case true:
{
logger.LogInformation("Open connections checked successfully: {Message}", checkForOpenConnectionsResult.Message);
Expand All @@ -53,7 +53,7 @@ public async Task<DatabaseRestoreResult> RestoreCleanDatabase()

if (!consent)
{
return new DatabaseRestoreResult(true, "User chose not to drop db and create a clean-db");
return new DatabaseResult(true, "User chose not to drop db and create a clean-db");
}

var forceDropOpenConnectionsResult = await ForceDropOpenConnections(CONTAINER_NAME, HOSTNAME, USERNAME, PASSWORD, DB_NAME);
Expand Down Expand Up @@ -107,67 +107,120 @@ public async Task<DatabaseRestoreResult> RestoreCleanDatabase()
}
catch (Exception e)
{
return new DatabaseRestoreResult(false, e.Message, Exception: e);
return new DatabaseResult(false, e.Message, Exception: e);
}
finally
{
await StopTemporaryPostgresDockerContainer(CONTAINER_NAME);
}

return new DatabaseRestoreResult(true, "Database restored successfully");
return new DatabaseResult(true, "Database restored successfully");
}

private async Task<DatabaseRestoreResult> DropDatabase(string containerName, string password, string hostname, string username, string dbname)
public async Task<DatabaseResult> BackupDatabase(string outputDirName)
{
try
{
var createContainerResult = await CreateTemporaryPostgresDockerContainer(CONTAINER_NAME, PASSWORD, forceRecreate: true, outputDirName);

if (!createContainerResult.Status)
{
throw new InvalidOperationException(createContainerResult.Message, createContainerResult.Exception);
}

logger.LogInformation("Postgres container created successfully: {Message}", createContainerResult.Message);

var result = await DumpDatabase(CONTAINER_NAME, PASSWORD, HOSTNAME, USERNAME, DB_NAME, SNAPHOT_DB_NAME);

if (!result.Status)
{
throw new InvalidOperationException(result.Message);
}
}
catch (Exception e)
{
return new DatabaseResult(false, e.Message, Exception: e);
}
finally
{
await StopTemporaryPostgresDockerContainer(CONTAINER_NAME);
}


return new DatabaseResult(true, "Database backup completed successfully");
}


private async Task<DatabaseResult> DropDatabase(string containerName, string password, string hostname, string username, string dbname)
{
var command = $"docker exec --env PGPASSWORD=\"{password}\" {containerName} psql -h {hostname} -U {username} postgres -c \"DROP DATABASE IF EXISTS {dbname}\"";
return await processHelper.ExecuteProcess(command, processParams => processParams.Process.ExitCode == 0 &&
!string.IsNullOrWhiteSpace(processParams.Output) &&
!processParams.HasError);
}

private async Task<DatabaseRestoreResult> CreateDatabase(string containerName, string password, string hostname, string username, string dbname)
private async Task<DatabaseResult> DumpDatabase(string containerName, string password, string hostname, string username, string dbname, string backupFile)
{
var command = $"docker exec --env PGPASSWORD=\"{password}\" {containerName} pg_dump -h {hostname} -U {username} {dbname} -f /dump/{backupFile}";
return await processHelper.ExecuteProcess(command, processParams => processParams.Process.ExitCode == 0 &&
!processParams.HasError);
}

private async Task<DatabaseResult> CreateDatabase(string containerName, string password, string hostname, string username, string dbname)
{
var command = $"docker exec --env PGPASSWORD=\"{password}\" {containerName} psql -h {hostname} -U {username} postgres -c \"CREATE DATABASE {dbname}\"";
return await processHelper.ExecuteProcess(command, processParams => processParams.Process.ExitCode == 0 &&
!string.IsNullOrWhiteSpace(processParams.Output) &&
!processParams.HasError);
}

private async Task<DatabaseRestoreResult> AlterDatabase(string containerName, string password, string hostname, string username, string dbname)
private async Task<DatabaseResult> AlterDatabase(string containerName, string password, string hostname, string username, string dbname)
{
var command = $"docker exec --env PGPASSWORD=\"{password}\" {containerName} psql -h {hostname} -U {username} postgres -c \"ALTER DATABASE {dbname} OWNER TO {username};\"";
return await processHelper.ExecuteProcess(command, processParams => processParams.Process.ExitCode == 0 &&
!string.IsNullOrWhiteSpace(processParams.Output) &&
!processParams.HasError);
}

private async Task<DatabaseRestoreResult> RestoreDatabase(string containerName, string password, string hostname, string username, string dbname, string backupFile)
private async Task<DatabaseResult> RestoreDatabase(string containerName, string password, string hostname, string username, string dbname, string restoreFile)
{
var command = $"docker exec --env PGPASSWORD=\"{password}\" {containerName} psql -h {hostname} -U {username} {dbname} -f /dump/{backupFile}";
var command = $"docker exec --env PGPASSWORD=\"{password}\" {containerName} psql -h {hostname} -U {username} {dbname} -f /dump/{restoreFile}";
return await processHelper.ExecuteProcess(command, processParams => processParams.Process.ExitCode == 0 &&
!string.IsNullOrWhiteSpace(processParams.Output) &&
!processParams.HasError);
}

private async Task<DatabaseRestoreResult> CreateTemporaryPostgresDockerContainer(string containerName, string password)
private async Task<DatabaseResult> CreateTemporaryPostgresDockerContainer(string containerName, string password, bool forceRecreate = false, string? outputDirectory = null)
{
try
{
var result = await IsTemporaryContainerRunning(containerName);
if (result.Status) return result;
if (forceRecreate)
{
await StopTemporaryPostgresDockerContainer(containerName);
}
else
{
var result = await IsTemporaryContainerRunning(containerName);
if (result.Status) return result;

var directory = Path.Combine(AppContext.BaseDirectory, @"Config\Database\dump-files");
if (result.IsError)
{
return new DatabaseResult(false, result.Message);
}
}

var directory = outputDirectory ?? Path.Combine(AppContext.BaseDirectory, @"Config\Database\dump-files");
var command = $"docker run -d --rm --name {containerName} -v \"{directory}:/dump\" -e POSTGRES_PASSWORD=\"{password}\" postgres";
// docker run -d --rm --name $ContainerName -v "$PSScriptRoot\dump-files:/dump" -e POSTGRES_PASSWORD="admin" postgres
return await processHelper.ExecuteProcess(command, processParams => !string.IsNullOrEmpty(processParams.Output?.Trim()));
}
catch (Exception e)
{
return new DatabaseRestoreResult(false, e.Message, Exception: e);
return new DatabaseResult(false, e.Message, Exception: e);
}
}

private async Task<DatabaseRestoreResult> IsTemporaryContainerRunning(string containerName)
private async Task<DatabaseResult> IsTemporaryContainerRunning(string containerName)
{
try
{
Expand All @@ -178,7 +231,7 @@ private async Task<DatabaseRestoreResult> IsTemporaryContainerRunning(string con
}
catch (Exception e)
{
return new DatabaseRestoreResult(false, e.Message, Exception: e);
return new DatabaseResult(false, e.Message, Exception: e);
}
}

Expand All @@ -199,8 +252,7 @@ private async Task StopTemporaryPostgresDockerContainer(string containerName)
}
}


private async Task<DatabaseRestoreResult> CheckForOpenConnections(string containerName, string hostname, string username, string password, string dbName)
private async Task<DatabaseResult> CheckForOpenConnections(string containerName, string hostname, string username, string password, string dbName)
{
try
{
Expand All @@ -213,11 +265,11 @@ private async Task<DatabaseRestoreResult> CheckForOpenConnections(string contain
}
catch (Exception e)
{
return new DatabaseRestoreResult(false, e.Message, Exception: e);
return new DatabaseResult(false, e.Message, Exception: e);
}
}

private async Task<DatabaseRestoreResult> ForceDropOpenConnections(string containerName, string hostname, string username, string password, string dbName)
private async Task<DatabaseResult> ForceDropOpenConnections(string containerName, string hostname, string username, string password, string dbName)
{
try
{
Expand All @@ -230,7 +282,7 @@ private async Task<DatabaseRestoreResult> ForceDropOpenConnections(string contai
}
catch (Exception e)
{
return new DatabaseRestoreResult(false, e.Message, Exception: e);
return new DatabaseResult(false, e.Message, Exception: e);
}
}
}
3 changes: 2 additions & 1 deletion ...ts.Performance/tools/snapshot-creator-v2/Features/Create/Helper/IDatabaseRestoreHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,6 @@ namespace Backbone.ConsumerApi.Tests.Performance.SnapshotCreator.V2.Features.Cre

public interface IDatabaseRestoreHelper
{
Task<DatabaseRestoreResult> RestoreCleanDatabase();
Task<DatabaseResult> RestoreCleanDatabase();
Task<DatabaseResult> BackupDatabase(string outputDirName);
}
2 changes: 1 addition & 1 deletion ...rApi.Tests.Performance/tools/snapshot-creator-v2/Features/Create/Helper/IProcessHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,5 @@ namespace Backbone.ConsumerApi.Tests.Performance.SnapshotCreator.V2.Features.Cre

public interface IProcessHelper
{
Task<DatabaseRestoreResult> ExecuteProcess(string command, Predicate<ProcessParams> processPredicate);
Task<DatabaseResult> ExecuteProcess(string command, Predicate<ProcessParams> processPredicate);
}
8 changes: 5 additions & 3 deletions ...erApi.Tests.Performance/tools/snapshot-creator-v2/Features/Create/Helper/ProcessHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ namespace Backbone.ConsumerApi.Tests.Performance.SnapshotCreator.V2.Features.Cre

public class ProcessHelper : IProcessHelper
{
public async Task<DatabaseRestoreResult> ExecuteProcess(string command, Predicate<ProcessParams> processPredicate)
public async Task<DatabaseResult> ExecuteProcess(string command, Predicate<ProcessParams> processPredicate)
{
var psi = new ProcessStartInfo("cmd", $"/c {command}")
{
Expand All @@ -26,15 +26,17 @@ public async Task<DatabaseRestoreResult> ExecuteProcess(string command, Predicat

var status = processPredicate(new ProcessParams(process, output, hasError));

return new DatabaseRestoreResult(status, $"{output} {errorMessage}", IsError: hasError);
return new DatabaseResult(status, $"{output} {errorMessage}", IsError: hasError);
}

private static string GetErrorMessage(string error)
{
return !string.IsNullOrWhiteSpace(error)
? error.StartsWith("psql: error: connection to server", StringComparison.OrdinalIgnoreCase)
? $"Is enmeshed backbone postgres container running?{Environment.NewLine}{ERROR}: {error}"
: $"{Environment.NewLine}{ERROR}:{error}"
: error.Contains("error during connect", StringComparison.OrdinalIgnoreCase)
? $"Is docker running?{Environment.NewLine}{ERROR}: {error}"
: $"{Environment.NewLine}{ERROR}:{error}"
: string.Empty;
}
}
1 change: 1 addition & 0 deletions ...merApi.Tests.Performance/tools/snapshot-creator-v2/Features/Shared/Constants/Resources.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public static class Resources
public const string POOL_CONFIG_FILE_READ_ERROR = "Pool configuration could not be read.";
public const string POOL_CONFIG_FILE_NOT_FOUND_ERROR = "Pool configuration file not found.";
public const string SNAPSHOT_CREATION_SUCCEED_MESSAGE = "Pool configuration with relationships and messages created successfully.";
public const string CLEAN_DB_SUCCEED_MESSAGE = "Restore of Clean-DB completed successfully.";

public static string BuildErrorDetails<TResult>(string message, DomainIdentity? senderIdentity, DomainIdentity? recipientIdentity, ApiResponse<TResult>? apiResponse = null)
{
Expand Down
Loading

0 comments on commit 9a1c984

Please sign in to comment.