Skip to content

Commit

Permalink
Another attempt to fix sync bugs.
Browse files Browse the repository at this point in the history
  • Loading branch information
Ove Bakken committed Jan 9, 2021
1 parent f79b858 commit 1b8fd60
Show file tree
Hide file tree
Showing 6 changed files with 120 additions and 61 deletions.
21 changes: 12 additions & 9 deletions MyS3.Tests/MyS3Tests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,30 +71,33 @@ public void S3ObjectKeyWithMetadataTest()
{
string encryptionPassword = "This is a test encryption password!";

string testFilePath = @"Images\Wallpapers\Nature\&¤#¤&&¤¤4456TRTGBFGBG ERUYH¤JH.jpg";

DateTime lastModifiedUTC = DateTime.UtcNow
.AddSeconds(new Random().Next(0, int.MaxValue));

long decryptedSize = new Random().Next(1000, int.MaxValue);
string testFilePath = Path.GetFullPath("../../../test image.jpg");
byte[] testFileData = File.ReadAllBytes(testFilePath);
string fileHash = Tools.DataToHash(testFileData);
DateTime lastModifiedUTC = File.GetLastWriteTimeUtc(testFilePath);

// ---

S3ObjectMetadata testS3ObjectMetadata = new S3ObjectMetadata(testFilePath, lastModifiedUTC, decryptedSize, encryptionPassword);
S3ObjectMetadata testS3ObjectMetadata = new S3ObjectMetadata(testFilePath, fileHash, testFileData.Length, lastModifiedUTC, encryptionPassword);
string testS3ObjectKeyWithMetadata = testS3ObjectMetadata.ToString();
output.WriteLine("Generated S3 object key with metadata: " + testS3ObjectKeyWithMetadata);

output.WriteLine("Generated S3 object key with metadata:");
output.WriteLine(testS3ObjectKeyWithMetadata);

Assert.True(S3ObjectMetadata.IsValidS3ObjectKeyWithMetadata(testS3ObjectKeyWithMetadata));


// ---

S3ObjectMetadata reconstructedTestS3ObjectKey = new S3ObjectMetadata(testS3ObjectKeyWithMetadata, encryptionPassword);

Assert.Equal(testFilePath, reconstructedTestS3ObjectKey.OfflineFilePathInsideMyS3);
Assert.Equal(fileHash, reconstructedTestS3ObjectKey.FileHash);
Assert.Equal(
lastModifiedUTC.ToLongDateString() + " " + lastModifiedUTC.ToLongTimeString(),
reconstructedTestS3ObjectKey.LastModifiedUTC.ToLongDateString() + " " + reconstructedTestS3ObjectKey.LastModifiedUTC.ToLongTimeString()
);
Assert.Equal(decryptedSize, reconstructedTestS3ObjectKey.DecryptedSize);
Assert.Equal(testFileData.Length, reconstructedTestS3ObjectKey.DecryptedSize);
}
}
}
Binary file added MyS3.Tests/test image.jpg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
8 changes: 4 additions & 4 deletions MyS3/MyS3.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk" ToolsVersion="Current">
<Project Sdk="Microsoft.NET.Sdk" ToolsVersion="Current">
<PropertyGroup>
<OutputType>Library</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
<ApplicationManifest>app.manifest</ApplicationManifest>
<Version>4.2.0</Version>
<Version>5.0.0</Version>
<Authors>Ove Bakken</Authors>
<Company>Ove Bakken</Company>
<Copyright>MIT</Copyright>
<PackageLicenseExpression>MIT</PackageLicenseExpression>
<FileUpgradeFlags>40</FileUpgradeFlags>
<UpgradeBackupLocation>D:\Arbeid\Utvikling\MyS3\MyS3\Backup\</UpgradeBackupLocation>
<OldToolsVersion>2.0</OldToolsVersion>
<AssemblyVersion>4.2.0.0</AssemblyVersion>
<FileVersion>4.2.0.0</FileVersion>
<AssemblyVersion>5.0.0.0</AssemblyVersion>
<FileVersion>5.0.0.0</FileVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.Core" Version="3.3.107.40" />
Expand Down
84 changes: 56 additions & 28 deletions MyS3/MyS3Runner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,18 @@
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Security.Cryptography;
using EncryptionAndHashingLibrary;

using Amazon;
using Amazon.S3;
using Amazon.S3.Model;

using EncryptionAndHashingLibrary;

namespace MyS3
{
public class MyS3Runner : IDisposable
Expand Down Expand Up @@ -266,7 +266,7 @@ public void Stop()

// ---

private Dictionary<string, DateTime> myS3FileIndexDict = new Dictionary<string, DateTime>(); // MyS3 file path ==> time last changed UTC
private Dictionary<string, DateTime> myS3FileIndexDict = new Dictionary<string, DateTime>(); // MyS3 file path ==> time last modified UTC
private Dictionary<string, S3ObjectMetadata> s3ObjectIndexDict = new Dictionary<string, S3ObjectMetadata>(); // S3 object key ==> S3 object metadata

public ImmutableList<string> UploadList
Expand Down Expand Up @@ -880,16 +880,22 @@ private void StartComparingS3AndMyS3()
// Check every removed S3 object (delete marker) against file index
foreach (S3ObjectMetadata removedS3ObjectWithMetadata in removedS3ObjectsList)
{
// Get paths and last modified
// Get paths
string offlineFilePathInsideMyS3 = removedS3ObjectWithMetadata.OfflineFilePathInsideMyS3;
string offlineFilePath = myS3Path + offlineFilePathInsideMyS3;
DateTime removedS3ObjectLastModifiedUTC = removedS3ObjectWithMetadata.LastModifiedUTC;
// Skip if file doesn't exist (perhaps already removed)
if (!File.Exists(offlineFilePath)) continue;
// Get file hashes
string offlineFileHash = Tools.DataToHash(File.ReadAllBytes(offlineFilePath));
string removedS3ObjectFileHash = removedS3ObjectWithMetadata.FileHash;
// Remove local file if deleted
if (copiedMyS3FileIndexDict.ContainsKey(offlineFilePathInsideMyS3) && // File removed on a different client
!s3ObjectIndexDict.ContainsKey(offlineFilePathInsideMyS3) && // File not overwritten
removedS3ObjectLastModifiedUTC == copiedMyS3FileIndexDict[offlineFilePathInsideMyS3]) // Same last modified time = same file
{ // = should be removed locally
if (copiedMyS3FileIndexDict.ContainsKey(offlineFilePathInsideMyS3) && // File removed on a different client
!s3ObjectIndexDict.ContainsKey(offlineFilePathInsideMyS3) && // File not in index = file removed and not overwritten
removedS3ObjectFileHash == offlineFileHash) // Same file contents
{ // = should be removed locally
// Remove file
try { File.Delete(offlineFilePath); } catch (Exception) { } // trigger file remove handler with necessary actions
Expand Down Expand Up @@ -930,14 +936,16 @@ private void StartComparingS3AndMyS3()
// Offline file exists
if (myS3FileIndexDict.ContainsKey(offlineFilePathInsideMyS3))
{
// Last modified
// Hash and last modified
DateTime offlineFileLastModifiedUTC;
string offlineFileHash = Tools.DataToHash(File.ReadAllBytes(offlineFilePath));
lock (myS3FileIndexDict)
offlineFileLastModifiedUTC = myS3FileIndexDict[offlineFilePathInsideMyS3];
DateTime s3ObjectLastModifiedUTC = s3ObjectIndexEntryKVP.Value.LastModifiedUTC;
string s3ObjectFileHash = s3ObjectIndexEntryKVP.Value.FileHash;
// When S3 object is new
if (s3ObjectLastModifiedUTC > offlineFileLastModifiedUTC.AddSeconds(5))
// When S3 object is new(er)
if (s3ObjectLastModifiedUTC > offlineFileLastModifiedUTC && s3ObjectFileHash != offlineFileHash)
{
// Skip if in use
if (Tools.IsFileLocked(offlineFilePath)) continue;
Expand Down Expand Up @@ -1001,12 +1009,14 @@ private void StartComparingS3AndMyS3()
// S3 object already exists
if (s3ObjectIndexDict.ContainsKey(offlineFilePathInsideMyS3))
{
// Last change
// Hash and last modified
DateTime offlineFileLastModifiedUTC = myS3FileIndexEntryKVP.Value;
string offlineFileHash = Tools.DataToHash(File.ReadAllBytes(offlineFilePath));
DateTime s3ObjectLastModifiedUTC = s3ObjectIndexDict[offlineFilePathInsideMyS3].LastModifiedUTC;
string s3ObjectFileHash = s3ObjectIndexDict[offlineFilePathInsideMyS3].FileHash;
// Newer file locally
if (offlineFileLastModifiedUTC.AddSeconds(-5) > s3ObjectLastModifiedUTC) // local modified time may be off by a few seconds
if (offlineFileLastModifiedUTC > s3ObjectLastModifiedUTC && offlineFileHash != s3ObjectFileHash)
lock (uploadListHashSet)
lock (downloadListHashSet)
lock (renameListDict)
Expand All @@ -1016,6 +1026,9 @@ private void StartComparingS3AndMyS3()
!renameListDict.Values.Contains(offlineFilePathInsideMyS3) && // not set to be renamed
!removeListHashSet.Contains(offlineFilePathInsideMyS3)) // not set to be removed
{
verboseLogFunc(offlineFileLastModifiedUTC.ToLongDateString() + " " + offlineFileLastModifiedUTC.ToLongTimeString() + " is newer than " +
s3ObjectLastModifiedUTC.ToLongDateString() + " " + s3ObjectLastModifiedUTC.ToLongTimeString() + ". Also " + offlineFileHash + " is not equal to " + s3ObjectFileHash);
// Add to upload queue
uploadListHashSet.Add(offlineFilePathInsideMyS3);
Expand Down Expand Up @@ -1170,10 +1183,11 @@ private void StartUploadWorker()
try
{
// 1. S3 object file data (encrypted)
byte[] fileData = File.ReadAllBytes(offlineFilePath);
File.WriteAllBytes(
encryptedUploadFilePath,
AesEncryptionWrapper.EncryptWithGCM(
File.ReadAllBytes(offlineFilePath),
fileData,
EncryptionAndHashingLibrary.Tools.GetPasswordAsEncryptionKey(
AesEncryptionWrapper.GCM_KEY_SIZE, encryptionPassword)
)
Expand Down Expand Up @@ -1209,8 +1223,9 @@ private void StartUploadWorker()
// 2. S3 object metadata
S3ObjectMetadata newS3ObjectMetadata = new S3ObjectMetadata(
offlineFilePathInsideMyS3,
Tools.DataToHash(fileData),
fileData.Length,
newOfflineFileLastModifiedUTC,
new FileInfo(offlineFilePath).Length,
encryptionPassword);
// Set progress info
Expand All @@ -1220,7 +1235,7 @@ private void StartUploadWorker()
if (verboseLogFunc != null)
verboseLogFunc("File \"" + offlineFilePathInsideMyS3.Replace(@"\", @" \ ").Replace(@"/", @" / ") +
"\" [" + Tools.GetFileSizeAsText(encryptedUploadFilePath) + "] starts uploading [" +
"\" [" + Tools.GetFileSizeAsText(encryptedUploadFilePath) + "] starts uploading [" +
uploadCounter + "/" + (uploadCounter + remainingUploads - 1) + "]");
// 3. Start upload
Expand All @@ -1237,7 +1252,7 @@ private void StartUploadWorker()
catch (Exception)
{
string problem = "File \"" + offlineFilePathInsideMyS3.Replace(@"\", @" \ ").Replace(@"/", @" / ") +
"\" upload failed or was aborted [" + uploadCounter + "/" +
"\" upload failed or was aborted [" + uploadCounter + "/" +
(uploadCounter + uploadListHashSet.Count - 1) + "]";
if (verboseLogFunc != null) verboseLogFunc(problem);
Expand Down Expand Up @@ -1313,9 +1328,11 @@ private void StartUploadWorker()
if (s3ObjectIndexDict.ContainsKey(offlineFilePathInsideMyS3))
{
// Very important: Remove older S3 object with same offline file path inside MyS3
try {
try
{
s3.RemoveAsync(s3ObjectIndexDict[offlineFilePathInsideMyS3].ToString(), null).Wait();
} catch (Exception) { }
}
catch (Exception) { }
s3ObjectIndexDict[offlineFilePathInsideMyS3] = newS3ObjectMetadata;
}
Expand Down Expand Up @@ -1345,6 +1362,11 @@ private void StartUploadWorker()
uploadCounter++;
}
}
catch (IOException)
{
// Exception: "The process cannot access the file '<file path>' because it is being used by another process.
// Basically just ignore it and let MyS3 retry later.
}
catch (Exception ex)
{
string problem = "File \"" + offlineFilePathInsideMyS3.Replace(@"\", @" \ ").Replace(@"/", @" / ") +
Expand Down Expand Up @@ -1511,13 +1533,13 @@ private void StartDownloadWorker()
// 2. Download complete
if (downloadTask.IsCompletedSuccessfully)
{
// Decrypt file and resize it's array
// Decrypt file and resize its array
byte[] decryptedFileData = AesEncryptionWrapper.DecryptForGCM(
File.ReadAllBytes(encryptedDownloadFilePath),
EncryptionAndHashingLibrary.Tools.GetPasswordAsEncryptionKey(AesEncryptionWrapper.GCM_KEY_SIZE, encryptionPassword)
);
long correctFileDataLength = s3ObjectIndexDict[offlineFilePathInsideMyS3].DecryptedSize;
byte[] fileData = new byte[correctFileDataLength];
long offlineFileCorrectNumberOfBytes = s3ObjectIndexDict[offlineFilePathInsideMyS3].DecryptedSize;
byte[] fileData = new byte[offlineFileCorrectNumberOfBytes];
Array.Copy(decryptedFileData, 0, fileData, 0, fileData.Length);
// ---
Expand All @@ -1541,7 +1563,7 @@ private void StartDownloadWorker()
// ---
// Finally create the file
// Finally create the file and attempt to set correct last modified time
File.WriteAllBytes(offlineFilePath, fileData);
File.SetLastWriteTimeUtc(offlineFilePath, s3ObjectlastModifiedUTC);
Expand Down Expand Up @@ -1585,6 +1607,11 @@ private void StartDownloadWorker()
if (verboseLogFunc != null) verboseLogFunc(problem);
errorLogFunc(myS3Path + RELATIVE_LOCAL_MYS3_LOG_DIRECTORY_PATH + "Decryption.log", problem);
}
catch (IOException)
{
// Exception: "The process cannot access the file '<file path>' because it is being used by another process.
// Basically just ignore it and let MyS3 retry later.
}
catch (Exception ex)
{
string problem = "S3 object for file \"" + offlineFilePathInsideMyS3.Replace(@"\", @" \ ").Replace(@"/", @" / ") +
Expand Down Expand Up @@ -1652,8 +1679,9 @@ private void StartRenameWorker()
S3ObjectMetadata oldS3ObjectMetadata = s3ObjectIndexDict[oldOfflineFilePathInsideMyS3];
S3ObjectMetadata newS3ObjectMetadata = new S3ObjectMetadata(
newOfflineFilePathInsideMyS3,
oldS3ObjectMetadata.LastModifiedUTC,
oldS3ObjectMetadata.DecryptedSize,
oldS3ObjectMetadata.FileHash, // file contents not changed when file renamed = same hash
oldS3ObjectMetadata.DecryptedSize, // and also the same size
oldS3ObjectMetadata.LastModifiedUTC, // and also the same modified time
encryptionPassword
);
Expand Down Expand Up @@ -2097,8 +2125,8 @@ private void RestoreFileVersions()
File.ReadAllBytes(encryptedDownloadFilePath),
EncryptionAndHashingLibrary.Tools.GetPasswordAsEncryptionKey(AesEncryptionWrapper.GCM_KEY_SIZE, encryptionPassword)
);
long correctFileDataLength = s3ObjectMetadata.DecryptedSize;
byte[] fileData = new byte[correctFileDataLength];
long offlineFileCorrectNumberOfBytes = s3ObjectMetadata.DecryptedSize;
byte[] fileData = new byte[offlineFileCorrectNumberOfBytes];
Array.Copy(decryptedFileData, 0, fileData, 0, fileData.Length);

// Create necessary directories
Expand Down
Loading

0 comments on commit 1b8fd60

Please sign in to comment.