From a34ab64c4c71358a61e56f52b858dfa99a82cee1 Mon Sep 17 00:00:00 2001 From: "qi.xu" Date: Tue, 19 Dec 2017 13:44:00 -0800 Subject: [PATCH 1/8] add a new API for resumable upload Add "PutObjectResult ResumableUploadObject(UploadObjectRequest req)" --- sdk/Common/ClientConfiguration.cs | 37 ----- sdk/Common/ResumableDownloadManager.cs | 4 +- sdk/Common/ResumableUploadManager.cs | 23 ++- sdk/Domain/DownloadObjectRequest.cs | 18 ++- sdk/Domain/UploadObjectRequest.cs | 145 ++++++++++++++++++ sdk/IOss.cs | 7 + sdk/OssClient.cs | 81 +++++++--- sdk/aliyun-oss-sdk.csproj | 1 + .../ObjectResumableOperationTest.cs | 57 +++---- 9 files changed, 278 insertions(+), 95 deletions(-) create mode 100644 sdk/Domain/UploadObjectRequest.cs diff --git a/sdk/Common/ClientConfiguration.cs b/sdk/Common/ClientConfiguration.cs index 88cba19..85df04f 100644 --- a/sdk/Common/ClientConfiguration.cs +++ b/sdk/Common/ClientConfiguration.cs @@ -30,8 +30,6 @@ public class ClientConfiguration private long _progressUpdateInterval = 1024 * 4; private long _directWriteStreamThreshold = 0; private long _maxPartCachingSize = 1024 * 1024 * 100; - private int _maxResumableUploadThreads = 8; - private int _maxResumableDownloadThreads = 8; private int _preReadBufferCount = 8; private bool _useSingleThreadReadInResumableUpload = false; @@ -197,25 +195,6 @@ public long MaxPartCachingSize } } - /// - /// Gets or sets the max uploading threads per resumable upload call - /// In multipart upload (resumable upload), by default it's multithreaded upload. You can specify the max thread count used per call. - /// If the number is no more than 1, then use single thread. - /// - /// The size of the max part caching. - public int MaxResumableUploadThreads - { - get - { - return _maxResumableUploadThreads; - } - set - { - _maxResumableUploadThreads = value; - - } - } - /// /// Gets or sets the pre read buffer count in resumable upload. /// The max value could be the same size of MaxResumableUploadThreads; @@ -250,22 +229,6 @@ public bool UseSingleThreadReadInResumableUpload } } - /// - /// Gets or sets the max resumable download threads. - /// - /// The max resumable download threads. - public int MaxResumableDownloadThreads - { - get - { - return _maxResumableDownloadThreads; - } - set - { - _maxResumableDownloadThreads = value; - } - } - /// /// Gets the default user agent /// diff --git a/sdk/Common/ResumableDownloadManager.cs b/sdk/Common/ResumableDownloadManager.cs index badf9cc..2ca44b7 100644 --- a/sdk/Common/ResumableDownloadManager.cs +++ b/sdk/Common/ResumableDownloadManager.cs @@ -66,7 +66,7 @@ private void DoResumableDownload(DownloadObjectRequest request, ResumableDownloa { // use single thread if MaxResumableUploadThreads is no bigger than 1 // or the part size is bigger than the conf.MaxPartCachingSize - if (resumableContext.PartContextList[0].Length > _conf.MaxPartCachingSize || _conf.MaxResumableDownloadThreads <= 1) + if (resumableContext.PartContextList[0].Length > _conf.MaxPartCachingSize || request.ParallelThreadCount <= 1) { DoResumableDownloadSingleThread(request, resumableContext, downloadProgressCallback); } @@ -185,7 +185,7 @@ private void DoResumableDownloadMultiThread(DownloadObjectRequest request, Resum } Exception e = null; - int parallel = Math.Min(Math.Min(_conf.MaxResumableDownloadThreads, resumableContext.PartContextList.Count), Environment.ProcessorCount); + int parallel = Math.Min(Math.Min(request.ParallelThreadCount, resumableContext.PartContextList.Count), Environment.ProcessorCount); ManualResetEvent[] taskFinishedEvents = new ManualResetEvent[parallel]; DownloadTaskParam[] taskParams = new DownloadTaskParam[parallel]; int nextPart = 0; diff --git a/sdk/Common/ResumableUploadManager.cs b/sdk/Common/ResumableUploadManager.cs index 93536c5..579769f 100644 --- a/sdk/Common/ResumableUploadManager.cs +++ b/sdk/Common/ResumableUploadManager.cs @@ -28,6 +28,7 @@ internal class ResumableUploadManager private EventHandler _uploadProgressCallback; private long _incrementalUploadedBytes; private object _callbackLock = new object(); + private UploadObjectRequest _request; public ResumableUploadManager(OssClient ossClient, int maxRetryTimes, ClientConfiguration conf) { @@ -36,16 +37,17 @@ public ResumableUploadManager(OssClient ossClient, int maxRetryTimes, ClientConf this._conf = conf; } - public void ResumableUploadWithRetry(string bucketName, string key, Stream content, ResumableContext resumableContext, - EventHandler uploadProgressCallback) + public void ResumableUploadWithRetry(UploadObjectRequest request, ResumableContext resumableContext) { - using (var fs = content) + _request = request; + Stream fs = request.UploadStream ?? new FileStream(request.UploadFile, FileMode.Open, FileAccess.Read, FileShare.Read); + try { for (int i = 0; i < _maxRetryTimes; i++) { try { - DoResumableUpload(bucketName, key, resumableContext, fs, uploadProgressCallback); + DoResumableUpload(request.BucketName, request.Key, resumableContext, fs, request.StreamTransferProgress); break; } catch (Exception ex) @@ -62,6 +64,13 @@ public void ResumableUploadWithRetry(string bucketName, string key, Stream conte } } } + finally + { + if(!object.ReferenceEquals(fs, request.UploadStream)) + { + fs.Dispose(); + } + } } private void DoResumableUpload(string bucketName, string key, ResumableContext resumableContext, Stream fs, @@ -71,7 +80,7 @@ private void DoResumableUpload(string bucketName, string key, ResumableContext r // use single thread if MaxResumableUploadThreads is no bigger than 1 // or when the stream is not file stream and the part size is bigger than the conf.MaxPartCachingSize - if (_conf.MaxResumableUploadThreads <= 1 || (!isFileStream || _conf.UseSingleThreadReadInResumableUpload) && resumableContext.PartContextList[0].Length > _conf.MaxPartCachingSize) + if (_request.ParallelThreadCount <= 1 || (!isFileStream || _conf.UseSingleThreadReadInResumableUpload) && resumableContext.PartContextList[0].Length > _conf.MaxPartCachingSize) { DoResumableUploadSingleThread(bucketName, key, resumableContext, fs, uploadProgressCallback); } @@ -322,7 +331,7 @@ private void DoResumableUploadFileMultiThread(string bucketName, string key, Res _incrementalUploadedBytes = 0; Exception e = null; - int parallel = Math.Min(_conf.MaxResumableUploadThreads, resumableContext.PartContextList.Count); + int parallel = Math.Min(_request.ParallelThreadCount, resumableContext.PartContextList.Count); ManualResetEvent[] taskFinishEvents = new ManualResetEvent[parallel]; UploadTask[] runningTasks = new UploadTask[parallel]; @@ -420,7 +429,7 @@ private void DoResumableUploadPreReadMultiThread(string bucketName, string key, _incrementalUploadedBytes = 0; Exception e = null; - int parallel = Math.Min(_conf.MaxResumableUploadThreads, resumableContext.PartContextList.Count); + int parallel = Math.Min(_request.ParallelThreadCount, resumableContext.PartContextList.Count); int preReadPartCount = Math.Min(parallel, _conf.PreReadBufferCount) + parallel; diff --git a/sdk/Domain/DownloadObjectRequest.cs b/sdk/Domain/DownloadObjectRequest.cs index f87bc06..734328d 100644 --- a/sdk/Domain/DownloadObjectRequest.cs +++ b/sdk/Domain/DownloadObjectRequest.cs @@ -17,6 +17,7 @@ public class DownloadObjectRequest private readonly IList _matchingETagConstraints = new List(); private readonly IList _nonmatchingEtagConstraints = new List(); private readonly ResponseHeaderOverrides _responseHeaders = new ResponseHeaderOverrides(); + private int _parallelThreadCount = 2; public DownloadObjectRequest(string bucketName, string key, string downloadFile) { @@ -25,6 +26,11 @@ public DownloadObjectRequest(string bucketName, string key, string downloadFile) DownloadFile = downloadFile; } + public DownloadObjectRequest(string bucketName, string key, string downloadFile, string checkpointDir) : this(bucketName, key, downloadFile) + { + CheckpointDir = checkpointDir; + } + /// /// Gets or sets the name of the bucket. /// @@ -69,10 +75,16 @@ public long? PartSize /// Gets or sets the parallel thread count. /// /// The parallel thread count. - public int? ParallelThreadCount + public int ParallelThreadCount { - get; - set; + get + { + return _parallelThreadCount; + } + set + { + _parallelThreadCount = value; + } } /// diff --git a/sdk/Domain/UploadObjectRequest.cs b/sdk/Domain/UploadObjectRequest.cs new file mode 100644 index 0000000..ee6d41a --- /dev/null +++ b/sdk/Domain/UploadObjectRequest.cs @@ -0,0 +1,145 @@ +/* + * Copyright (C) Alibaba Cloud Computing + * All rights reserved. + * + */ + +using System; +using System.Collections.Generic; +using System.IO; + +using Aliyun.OSS.Util; +using Aliyun.OSS.Common.Internal; +namespace Aliyun.OSS +{ + /// + /// Upload object request. + /// + public class UploadObjectRequest + { + private int _parallelThreadCount = 2; + + /// + /// Initializes a new instance of the class. + /// + /// Bucket name. + /// Key. + /// Upload file. + public UploadObjectRequest(string bucketName, string key, string uploadFile) + { + BucketName = bucketName; + Key = key; + UploadFile = uploadFile; + } + + /// + /// Initializes a new instance of the class. + /// + /// Bucket name. + /// Key. + /// Upload stream. + public UploadObjectRequest(string bucketName, string key, Stream uploadStream) + { + BucketName = bucketName; + Key = key; + UploadStream = uploadStream; + } + + /// + /// Gets or sets the name of the bucket. + /// + /// The name of the bucket. + public string BucketName + { + get; + set; + } + + /// + /// Gets or sets the key. + /// + /// The key. + public string Key + { + get; + set; + } + + /// + /// Gets or sets the upload file. + /// + /// The upload file. + public string UploadFile + { + get; + set; + } + + /// + /// Gets or sets the upload stream. + /// Note: when both UploadStream and UploadFile properties are set, the UploadStream will be used. + /// + /// The upload stream. + public Stream UploadStream + { + get; + set; + } + + /// + /// Gets or sets the size of the part. + /// + /// The size of the part. + public long? PartSize + { + get; + set; + } + + /// + /// Gets or sets the parallel thread count. + /// + /// The parallel thread count. + public int ParallelThreadCount + { + get + { + return _parallelThreadCount; + } + set + { + _parallelThreadCount = value; + } + } + + /// + /// Gets or sets the checkpoint dir. + /// + /// The checkpoint dir. + public string CheckpointDir + { + get; + set; + } + + /// + /// Gets or sets the stream transfer progress. + /// + /// The stream transfer progress. + public EventHandler StreamTransferProgress + { + get; + set; + } + + /// + /// Gets or sets the metadata. + /// + /// The metadata. + public ObjectMetadata Metadata + { + get; + set; + } + } +} diff --git a/sdk/IOss.cs b/sdk/IOss.cs index 775f7f9..f368614 100644 --- a/sdk/IOss.cs +++ b/sdk/IOss.cs @@ -496,6 +496,13 @@ PutObjectResult ResumableUploadObject(string bucketName, string key, string file PutObjectResult ResumableUploadObject(string bucketName, string key, Stream content, ObjectMetadata metadata, string checkpointDir, long? partSize = null, EventHandler streamTransferProgress = null); + /// + /// Resumables the upload object. + /// + /// The upload object. + /// Upload Request. + PutObjectResult ResumableUploadObject(UploadObjectRequest request); + /// /// Appends object to OSS according to the /// diff --git a/sdk/OssClient.cs b/sdk/OssClient.cs index 62cc31c..ad6f806 100644 --- a/sdk/OssClient.cs +++ b/sdk/OssClient.cs @@ -747,48 +747,92 @@ public PutObjectResult ResumableUploadObject(string bucketName, string key, stri public PutObjectResult ResumableUploadObject(string bucketName, string key, Stream content, ObjectMetadata metadata, string checkpointDir, long? partSize = null, EventHandler streamTransferProgress = null) { - ThrowIfNullRequest(bucketName); - ThrowIfNullRequest(key); - ThrowIfNullRequest(content); + UploadObjectRequest request = new UploadObjectRequest(bucketName, key, content); + request.CheckpointDir = checkpointDir; + request.PartSize = partSize; + request.StreamTransferProgress = streamTransferProgress; + request.Metadata = metadata; - if (!content.CanSeek) + return ResumableUploadObject(request); + } + + public PutObjectResult ResumableUploadObject(UploadObjectRequest request) + { + ThrowIfNullRequest(request); + ThrowIfNullRequest(request.BucketName); + ThrowIfNullRequest(request.Key); + if (string.IsNullOrEmpty(request.UploadFile) && request.UploadStream == null) + { + throw new ArgumentException("Parameter request.UploadFile or request.UploadStream must not be null."); + } + + if (request.UploadStream != null && !request.UploadStream.CanSeek) { - throw new ArgumentException("Parameter content must be seekable---for nonseekable stream, please call UploadObject instead."); + throw new ArgumentException("Parameter request.UploadStream must be seekable---for nonseekable stream, please call UploadObject instead."); } // calculates content-type - metadata = metadata ?? new ObjectMetadata(); - SetContentTypeIfNull(key, null, ref metadata); + if (request.Metadata == null) + { + request.Metadata = new ObjectMetadata(); + } + + ObjectMetadata metadata = request.Metadata; + SetContentTypeIfNull(request.Key, null, ref metadata); // Adjust part size - long actualPartSize = AdjustPartSize(partSize); + long actualPartSize = AdjustPartSize(request.PartSize); // If the file size is less than the part size, upload it directly. - if (content.Length <= actualPartSize) + long fileSize = 0; + Stream uploadSteam = null; + + if (request.UploadStream != null) { - var putObjectRequest = new PutObjectRequest(bucketName, key, content, metadata) + fileSize = request.UploadStream.Length; + uploadSteam = request.UploadStream; + } + else + { + fileSize = new System.IO.FileInfo(request.UploadFile).Length; + uploadSteam = new FileStream(request.UploadFile, FileMode.Open, FileAccess.Read, FileShare.Read); + } + + if (fileSize <= actualPartSize) + { + try { - StreamTransferProgress = streamTransferProgress, - }; - return PutObject(putObjectRequest); + var putObjectRequest = new PutObjectRequest(request.BucketName, request.Key, uploadSteam, metadata) + { + StreamTransferProgress = request.StreamTransferProgress, + }; + return PutObject(putObjectRequest); + } + finally + { + if (!object.ReferenceEquals(uploadSteam, request.UploadStream)) + { + uploadSteam.Dispose(); + } + } } - var resumableContext = LoadResumableUploadContext(bucketName, key, content, - checkpointDir, actualPartSize); + var resumableContext = LoadResumableUploadContext(request.BucketName, request.Key, uploadSteam, + request.CheckpointDir, actualPartSize); if (resumableContext.UploadId == null) { - var initRequest = new InitiateMultipartUploadRequest(bucketName, key, metadata); + var initRequest = new InitiateMultipartUploadRequest(request.BucketName, request.Key, metadata); var initResult = InitiateMultipartUpload(initRequest); resumableContext.UploadId = initResult.UploadId; } int maxRetry = ((RetryableServiceClient)_serviceClient).MaxRetryTimes; ResumableUploadManager uploadManager = new ResumableUploadManager(this, maxRetry, OssUtils.GetClientConfiguration(_serviceClient)); - uploadManager.ResumableUploadWithRetry(bucketName, key, content, resumableContext, streamTransferProgress); + uploadManager.ResumableUploadWithRetry(request, resumableContext); // Completes the upload - var completeRequest = new CompleteMultipartUploadRequest(bucketName, key, resumableContext.UploadId); + var completeRequest = new CompleteMultipartUploadRequest(request.BucketName, request.Key, resumableContext.UploadId); if (metadata.HttpMetadata.ContainsKey(HttpHeaders.Callback)) { var callbackMetadata = new ObjectMetadata(); @@ -811,7 +855,6 @@ public PutObjectResult ResumableUploadObject(string bucketName, string key, Stre return result; } - /// public AppendObjectResult AppendObject(AppendObjectRequest request) { diff --git a/sdk/aliyun-oss-sdk.csproj b/sdk/aliyun-oss-sdk.csproj index 9178edb..05bfc90 100644 --- a/sdk/aliyun-oss-sdk.csproj +++ b/sdk/aliyun-oss-sdk.csproj @@ -381,6 +381,7 @@ + diff --git a/test/TestCase/ObjectTestCase/ObjectResumableOperationTest.cs b/test/TestCase/ObjectTestCase/ObjectResumableOperationTest.cs index 00c1e8a..93c59b7 100644 --- a/test/TestCase/ObjectTestCase/ObjectResumableOperationTest.cs +++ b/test/TestCase/ObjectTestCase/ObjectResumableOperationTest.cs @@ -138,21 +138,30 @@ override protected void ThrowIfNullRequest(TRequestType request) [Test] public void ResumableUploadObjectWithRetry() + { + ResumableUploadObjectWithRetry(2); + } + + public void ResumableUploadObjectWithRetry(int threadCount) { var key = OssTestUtils.GetObjectKey(_className); int failedCount = 0; + UploadObjectRequest request = new UploadObjectRequest(_bucketName, key, Config.MultiUploadTestFile); + request.CheckpointDir = Config.DownloadFolder; + request.PartSize = 200 * 1024; + request.StreamTransferProgress = (sender, e) => { + if (failedCount < 2) + { + failedCount++; + throw new Exception("injection failure"); + } + }; + request.ParallelThreadCount = threadCount; + try { - var result = _ossClient.ResumableUploadObject(_bucketName, key, Config.MultiUploadTestFile, null, - Config.DownloadFolder, 1024 * 200, - (sender, e) => - { - if (failedCount < 2) { - failedCount++; - throw new Exception("injection failure"); - } - }); + var result = _ossClient.ResumableUploadObject(request); Assert.IsTrue(OssTestUtils.ObjectExists(_ossClient, _bucketName, key)); Assert.IsTrue(result.ETag.Length > 0); } @@ -173,7 +182,7 @@ public void ResumableUploadObjectWithRetryUsingSingleReadThread() try { - ResumableUploadObjectWithRetry(); + ResumableUploadObjectWithRetry(2); } finally { @@ -184,17 +193,7 @@ public void ResumableUploadObjectWithRetryUsingSingleReadThread() [Test] public void ResumableUploadObjectWithRetryUsingSingleThread() { - int restoreVal = _config.MaxResumableUploadThreads; - _config.MaxResumableUploadThreads = 1; - - try - { - ResumableUploadObjectWithRetry(); - } - finally - { - _config.MaxResumableUploadThreads = restoreVal; - } + ResumableUploadObjectWithRetry(1); } [Test] @@ -501,9 +500,13 @@ public void ResumableUploadObjectTestWithSingleThread() var fileInfo = new FileInfo(Config.MultiUploadTestFile); var fileSize = fileInfo.Length; var config = new ClientConfiguration(); - config.MaxResumableUploadThreads = 1; var client = OssClientFactory.CreateOssClient(config); - var result = client.ResumableUploadObject(_bucketName, key, Config.MultiUploadTestFile, new ObjectMetadata(), Config.DownloadFolder, fileSize / 3); + UploadObjectRequest request = new UploadObjectRequest(_bucketName, key, Config.MultiUploadTestFile); + request.Metadata = new ObjectMetadata(); + request.CheckpointDir = Config.DownloadFolder; + request.PartSize = fileSize / 3; + request.ParallelThreadCount = 1; + var result = client.ResumableUploadObject(request); Assert.IsTrue(OssTestUtils.ObjectExists(_ossClient, _bucketName, key)); Assert.IsTrue(result.ETag.Length > 0); } @@ -654,11 +657,11 @@ public void ResumableDownloadBigFileSingleThreadTest() var key = OssTestUtils.GetObjectKey(_className); _ossClient.PutObject(_bucketName, key, Config.MultiUploadTestFile); var config = new ClientConfiguration(); - config.MaxResumableDownloadThreads = 1; var client = OssClientFactory.CreateOssClient(config); try { DownloadObjectRequest request = new DownloadObjectRequest(_bucketName, key, targetFile); + request.ParallelThreadCount = 1; var metadata = client.ResumableDownloadObject(request); var expectedETag = metadata.ETag; var downloadedFileETag = FileUtils.ComputeContentMd5(targetFile); @@ -687,12 +690,12 @@ public void ResumableDownloadBigFileSingleThreadWithMd5CheckTest() var key = OssTestUtils.GetObjectKey(_className); _ossClient.PutObject(_bucketName, key, Config.MultiUploadTestFile); var config = new ClientConfiguration(); - config.MaxResumableDownloadThreads = 1; config.EnalbeMD5Check = true; var client = OssClientFactory.CreateOssClient(config); try { DownloadObjectRequest request = new DownloadObjectRequest(_bucketName, key, targetFile); + request.ParallelThreadCount = 1; var metadata = client.ResumableDownloadObject(request); var expectedETag = metadata.ETag; var downloadedFileETag = FileUtils.ComputeContentMd5(targetFile); @@ -724,12 +727,12 @@ public void ResumableDownloadBigFileSingleThreadWithProgressUpdateTest() int percentDone = 0; long totalBytesDownloaded = 0; var config = new ClientConfiguration(); - config.MaxResumableDownloadThreads = 1; config.EnalbeMD5Check = true; var client = OssClientFactory.CreateOssClient(config); try { DownloadObjectRequest request = new DownloadObjectRequest(_bucketName, key, targetFile); + request.ParallelThreadCount = 1; request.StreamTransferProgress += (object sender, StreamTransferProgressArgs e) => { totalBytesDownloaded += e.TransferredBytes; Console.WriteLine("TotalBytes:" + e.TotalBytes); @@ -982,7 +985,7 @@ public void ResumableDownloadBigFileSingleThreadBreakAndResumeTest() }; var config = new ClientConfiguration(); - config.MaxResumableDownloadThreads = 1; + request.ParallelThreadCount = 1; var client = OssClientFactory.CreateOssClient(config); var metadata = client.ResumableDownloadObject(request); From 6b7d2064639ee89dc5c7a17abf710be90b9c6a36 Mon Sep 17 00:00:00 2001 From: qixu001 Date: Tue, 19 Dec 2017 14:57:21 -0800 Subject: [PATCH 2/8] use a different naming convention for ResumableDownloadContext to prevent naming conflict with upload. --- sdk/Domain/ResumableContext.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sdk/Domain/ResumableContext.cs b/sdk/Domain/ResumableContext.cs index 78009a8..0b727d6 100644 --- a/sdk/Domain/ResumableContext.cs +++ b/sdk/Domain/ResumableContext.cs @@ -69,11 +69,13 @@ public bool FromString(string value) IsCompleted = isCompleted; int partNum; - if (int.TryParse(tokens[4], out partNum) && partNum > 0 && !string.IsNullOrEmpty(tokens[5])) + if (!(int.TryParse(tokens[4], out partNum) && partNum > 0 && !string.IsNullOrEmpty(tokens[5]))) { - PartETag = new PartETag(partNum, tokens[5]); + return false; } + PartETag = new PartETag(partNum, tokens[5]); + return true; } @@ -283,7 +285,8 @@ override protected string GenerateCheckpointFile() public ResumableCopyContext (string sourceBucketName, string sourceKey, string destBucketName, string destKey, string checkpointDir) - : base(destBucketName, destKey, checkpointDir) + : base(destBucketName, + destKey, checkpointDir) { SourceBucketName = sourceBucketName; SourceKey = sourceKey; @@ -297,6 +300,10 @@ public ResumableDownloadContext(string bucketName, string key, string checkpoint { } + override protected string GenerateCheckpointFile() + { + return GetCheckpointFilePath(CheckpointDir, "_Download" + "_" + Base64(BucketName) + "_" + Base64(Key)); + } public long GetDownloadedBytes() { return GetUploadedBytes(); From 7c51f770147f1bf3fdc3c867f71ff627bfd3a9cd Mon Sep 17 00:00:00 2001 From: "qi.xu" Date: Tue, 19 Dec 2017 18:03:37 -0800 Subject: [PATCH 3/8] fix the async pattern issues 1) catch exceptions in async call's 2) Fix the async pattern; the following standard async pattern now works: var ar = Beginxxx(); Endxxx(ar); 3) Dispose the stream once the resumable upload API finishes. --- sdk/Common/Communication/ServiceClient.cs | 4 + sdk/Common/Communication/ServiceClientImpl.cs | 116 ++++++++++++------ sdk/Common/ResumableUploadManager.cs | 10 +- sdk/Domain/UploadObjectRequest.cs | 1 + sdk/IOss.cs | 3 +- sdk/OssClient.cs | 5 +- sdk/Util/OssUtils.cs | 45 +++++-- .../ObjectBasicOperationTest.cs | 59 +++++++++ 8 files changed, 186 insertions(+), 57 deletions(-) diff --git a/sdk/Common/Communication/ServiceClient.cs b/sdk/Common/Communication/ServiceClient.cs index 2c84aa5..e98e855 100644 --- a/sdk/Common/Communication/ServiceClient.cs +++ b/sdk/Common/Communication/ServiceClient.cs @@ -65,6 +65,10 @@ public ServiceResponse EndSend(IAsyncResult aysncResult) { var ar = aysncResult as AsyncResult; Debug.Assert(ar != null); + if (ar == null) + { + throw new ArgumentException("ar must be type of AsyncResult"); + } try { diff --git a/sdk/Common/Communication/ServiceClientImpl.cs b/sdk/Common/Communication/ServiceClientImpl.cs index 52a1ed2..4930b66 100644 --- a/sdk/Common/Communication/ServiceClientImpl.cs +++ b/sdk/Common/Communication/ServiceClientImpl.cs @@ -169,7 +169,7 @@ protected override ServiceResponse SendCore(ServiceRequest serviceRequest, ExecutionContext context) { var request = HttpFactory.CreateWebRequest(serviceRequest, Configuration); - SetRequestContent(request, serviceRequest, false, null, Configuration); + SetRequestContent(request, serviceRequest, Configuration); try { var response = request.GetResponse() as HttpWebResponse; @@ -194,8 +194,9 @@ protected override IAsyncResult BeginSendCore(ServiceRequest serviceRequest, Request = serviceRequest }; - SetRequestContent(request, serviceRequest, true, - () => request.BeginGetResponse(OnGetResponseCompleted, asyncResult), Configuration); + BeginSetRequestContent(request, serviceRequest, + () => request.BeginGetResponse(OnGetResponseCompleted, asyncResult), Configuration, + asyncResult); return asyncResult; } @@ -236,8 +237,9 @@ private void OnGetResponseCompleted(IAsyncResult ar) } } - private static void SetRequestContent(HttpWebRequest webRequest, ServiceRequest serviceRequest, - bool async, OssAction asyncCallback, ClientConfiguration clientConfiguration) + private static void SetRequestContent(HttpWebRequest webRequest, + ServiceRequest serviceRequest, + ClientConfiguration clientConfiguration) { var data = serviceRequest.BuildRequestContent(); @@ -245,10 +247,6 @@ private static void SetRequestContent(HttpWebRequest webRequest, ServiceRequest (serviceRequest.Method != HttpMethod.Put && serviceRequest.Method != HttpMethod.Post)) { - // Skip setting content body in this case. - if (async) - asyncCallback(); - return; } @@ -273,39 +271,87 @@ private static void SetRequestContent(HttpWebRequest webRequest, ServiceRequest } } - if (async) + using (var requestStream = webRequest.GetRequestStream()) { - webRequest.BeginGetRequestStream( - (ar) => - { - using (var requestStream = webRequest.EndGetRequestStream(ar)) - { - if (!webRequest.SendChunked) - { - IoUtils.WriteTo(data, requestStream, webRequest.ContentLength); - } - else - { - IoUtils.WriteTo(data, requestStream); - } - } - asyncCallback(); - }, null); + if (!webRequest.SendChunked) + { + IoUtils.WriteTo(data, requestStream, webRequest.ContentLength); + } + else + { + IoUtils.WriteTo(data, requestStream); + } + } + } + + private static void BeginSetRequestContent(HttpWebRequest webRequest, ServiceRequest serviceRequest, + OssAction asyncCallback, ClientConfiguration clientConfiguration, HttpAsyncResult result) + { + var data = serviceRequest.BuildRequestContent(); + + if (data == null || + (serviceRequest.Method != HttpMethod.Put && + serviceRequest.Method != HttpMethod.Post)) + { + // Skip setting content body in this case. + try + { + asyncCallback(); + } + catch(Exception e) + { + result.WebRequest.Abort(); + result.Complete(e); + } + + return; + } + + // Write data to the request stream. + long userSetContentLength = -1; + if (serviceRequest.Headers.ContainsKey(HttpHeaders.ContentLength)) + userSetContentLength = long.Parse(serviceRequest.Headers[HttpHeaders.ContentLength]); + + if (serviceRequest.UseChunkedEncoding || !data.CanSeek) // when data cannot seek, we have to use chunked encoding as there's no way to set the length + { + webRequest.SendChunked = true; + webRequest.AllowWriteStreamBuffering = false; // when using chunked encoding, the data is likely big and thus not use write buffer; } else { - using (var requestStream = webRequest.GetRequestStream()) + long streamLength = data.Length - data.Position; + webRequest.ContentLength = (userSetContentLength >= 0 && + userSetContentLength <= streamLength) ? userSetContentLength : streamLength; + if (webRequest.ContentLength > clientConfiguration.DirectWriteStreamThreshold) { - if (!webRequest.SendChunked) - { - IoUtils.WriteTo(data, requestStream, webRequest.ContentLength); - } - else - { - IoUtils.WriteTo(data, requestStream); - } + webRequest.AllowWriteStreamBuffering = false; } } + + webRequest.BeginGetRequestStream( + (ar) => + { + try + { + using (var requestStream = webRequest.EndGetRequestStream(ar)) + { + if (!webRequest.SendChunked) + { + IoUtils.WriteTo(data, requestStream, webRequest.ContentLength); + } + else + { + IoUtils.WriteTo(data, requestStream); + } + } + asyncCallback(); + } + catch(Exception e) + { + result.WebRequest.Abort(); + result.Complete(e); + } + }, null); } private static ServiceResponse HandleException(WebException ex) diff --git a/sdk/Common/ResumableUploadManager.cs b/sdk/Common/ResumableUploadManager.cs index 579769f..51f38f7 100644 --- a/sdk/Common/ResumableUploadManager.cs +++ b/sdk/Common/ResumableUploadManager.cs @@ -40,8 +40,7 @@ public ResumableUploadManager(OssClient ossClient, int maxRetryTimes, ClientConf public void ResumableUploadWithRetry(UploadObjectRequest request, ResumableContext resumableContext) { _request = request; - Stream fs = request.UploadStream ?? new FileStream(request.UploadFile, FileMode.Open, FileAccess.Read, FileShare.Read); - try + using (Stream fs = request.UploadStream ?? new FileStream(request.UploadFile, FileMode.Open, FileAccess.Read, FileShare.Read)) { for (int i = 0; i < _maxRetryTimes; i++) { @@ -64,13 +63,6 @@ public void ResumableUploadWithRetry(UploadObjectRequest request, ResumableConte } } } - finally - { - if(!object.ReferenceEquals(fs, request.UploadStream)) - { - fs.Dispose(); - } - } } private void DoResumableUpload(string bucketName, string key, ResumableContext resumableContext, Stream fs, diff --git a/sdk/Domain/UploadObjectRequest.cs b/sdk/Domain/UploadObjectRequest.cs index ee6d41a..6f64947 100644 --- a/sdk/Domain/UploadObjectRequest.cs +++ b/sdk/Domain/UploadObjectRequest.cs @@ -78,6 +78,7 @@ public string UploadFile /// /// Gets or sets the upload stream. /// Note: when both UploadStream and UploadFile properties are set, the UploadStream will be used. + /// It will be disposed once the ResumableUploadFile finishes. /// /// The upload stream. public Stream UploadStream diff --git a/sdk/IOss.cs b/sdk/IOss.cs index f368614..b00dc97 100644 --- a/sdk/IOss.cs +++ b/sdk/IOss.cs @@ -486,7 +486,7 @@ PutObjectResult ResumableUploadObject(string bucketName, string key, string file /// /// name /// - /// + /// . Content is disposed after the call finishes. /// metadata /// Check point dir. If it's not specified, then no checkpoint file is saved and thus resumable file upload is not supported. /// Part size. If it's not specified, or the size is smaller than @@ -498,6 +498,7 @@ PutObjectResult ResumableUploadObject(string bucketName, string key, Stream cont /// /// Resumables the upload object. + /// The request.UploadStream will be disposed once the call finishes. /// /// The upload object. /// Upload Request. diff --git a/sdk/OssClient.cs b/sdk/OssClient.cs index ad6f806..5aea852 100644 --- a/sdk/OssClient.cs +++ b/sdk/OssClient.cs @@ -810,10 +810,7 @@ public PutObjectResult ResumableUploadObject(UploadObjectRequest request) } finally { - if (!object.ReferenceEquals(uploadSteam, request.UploadStream)) - { - uploadSteam.Dispose(); - } + uploadSteam.Dispose(); } } diff --git a/sdk/Util/OssUtils.cs b/sdk/Util/OssUtils.cs index f39e901..52f53cc 100644 --- a/sdk/Util/OssUtils.cs +++ b/sdk/Util/OssUtils.cs @@ -345,11 +345,24 @@ internal static IAsyncResult BeginOperationHelper(TCommand cmd, AsyncC internal static TResult EndOperationHelper(IServiceClient serviceClient, IAsyncResult asyncResult) { var response = EndOperationHelper(serviceClient, asyncResult); - using (RetryableAsyncResult retryableAsyncResult = asyncResult as RetryableAsyncResult) + RetryableAsyncResult retryableAsyncResult = asyncResult as RetryableAsyncResult; + if (retryableAsyncResult == null) { - Debug.Assert(retryableAsyncResult != null); - OssCommand cmd = (OssCommand)retryableAsyncResult.Context.Command; - return cmd.DeserializeResponse(response); + retryableAsyncResult = asyncResult.AsyncState as RetryableAsyncResult; + } + + if (retryableAsyncResult != null) + { + using (retryableAsyncResult) + { + Debug.Assert(retryableAsyncResult != null); + OssCommand cmd = (OssCommand)retryableAsyncResult.Context.Command; + return cmd.DeserializeResponse(response); + } + } + else + { + throw new ArgumentException("asyncResult"); } } @@ -360,11 +373,27 @@ private static ServiceResponse EndOperationHelper(IServiceClient serviceClient, var retryableAsyncResult = asyncResult as RetryableAsyncResult; if (retryableAsyncResult == null) - throw new ArgumentException("retryableAsyncResult should not be null"); + { + ServiceClientImpl.HttpAsyncResult httpAsyncResult = + asyncResult as ServiceClientImpl.HttpAsyncResult; + if (httpAsyncResult == null) + { + throw new ArgumentException("asyncResult"); + } + + retryableAsyncResult = httpAsyncResult.AsyncState as RetryableAsyncResult; + } - ServiceClientImpl.HttpAsyncResult httpAsyncResult = - retryableAsyncResult.InnerAsyncResult as ServiceClientImpl.HttpAsyncResult; - return serviceClient.EndSend(httpAsyncResult); + if (retryableAsyncResult != null) + { + ServiceClientImpl.HttpAsyncResult httpAsyncResult = + retryableAsyncResult.InnerAsyncResult as ServiceClientImpl.HttpAsyncResult; + return serviceClient.EndSend(httpAsyncResult); + } + else + { + throw new ArgumentException("asyncResult"); + } } internal static void CheckCredentials(string accessKeyId, string accessKeySecret) diff --git a/test/TestCase/ObjectTestCase/ObjectBasicOperationTest.cs b/test/TestCase/ObjectTestCase/ObjectBasicOperationTest.cs index 59ef96c..90e057c 100644 --- a/test/TestCase/ObjectTestCase/ObjectBasicOperationTest.cs +++ b/test/TestCase/ObjectTestCase/ObjectBasicOperationTest.cs @@ -3,6 +3,7 @@ using System.IO; using Aliyun.OSS; using Aliyun.OSS.Common; +using Aliyun.OSS.Common.Communication; using Aliyun.OSS.Model; using Aliyun.OSS.Util; using Aliyun.OSS.Test.Util; @@ -107,6 +108,64 @@ public void UploadObjectBasicSettingsAsyncTest() } } + [Test] + public void UploadObjectBasicSettingsAsyncStandardUsageTest() + { + var key = OssTestUtils.GetObjectKey(_className); + + try + { + //upload the object + var ar = OssTestUtils.BeginUploadObject(_ossClient, _bucketName, key, + Config.UploadTestFile, + null, + null); + var result = OssTestUtils.EndUploadObject(_ossClient, ar); + Assert.AreEqual(result.HttpStatusCode, HttpStatusCode.OK); + Assert.IsTrue(OssTestUtils.ObjectExists(_ossClient, _bucketName, key)); + } + finally + { + if (OssTestUtils.ObjectExists(_ossClient, _bucketName, key)) + { + _ossClient.DeleteObject(_bucketName, key); + } + } + } + + [Test] + public void UploadObjectBasicSettingsAsyncExceptionTest() + { + var key = OssTestUtils.GetObjectKey(_className); + Exception e = new Exception("inject exception"); + try + { + //upload the object + AutoResetEvent done = new AutoResetEvent(false); + var result = OssTestUtils.BeginUploadObject(_ossClient, _bucketName, key, + Config.UploadTestFile, + (ar) => { + done.Set(); + throw e; + }, + null); + done.WaitOne(1000 * 100); // in real world, this is not needed. This is just to make sure the injected exception gets caught. + OssTestUtils.EndUploadObject(_ossClient, result); + } + catch(Exception ex) + { + Assert.AreSame(ex, e); + } + finally + { + if (OssTestUtils.ObjectExists(_ossClient, _bucketName, key)) + { + _ossClient.DeleteObject(_bucketName, key); + } + } + } + + [Test] public void UploadUnSeekableStreamTest() { From 279407682898b69ff5c68e3fb026fb49a58b0cba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=B8=A3=E9=95=9D?= Date: Thu, 21 Dec 2017 11:11:00 +0800 Subject: [PATCH 4/8] Refine default ParallelThreadCount to 3 --- sdk/Domain/DownloadObjectRequest.cs | 2 +- sdk/Domain/UploadObjectRequest.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/Domain/DownloadObjectRequest.cs b/sdk/Domain/DownloadObjectRequest.cs index 734328d..24c0640 100644 --- a/sdk/Domain/DownloadObjectRequest.cs +++ b/sdk/Domain/DownloadObjectRequest.cs @@ -17,7 +17,7 @@ public class DownloadObjectRequest private readonly IList _matchingETagConstraints = new List(); private readonly IList _nonmatchingEtagConstraints = new List(); private readonly ResponseHeaderOverrides _responseHeaders = new ResponseHeaderOverrides(); - private int _parallelThreadCount = 2; + private int _parallelThreadCount = 3; public DownloadObjectRequest(string bucketName, string key, string downloadFile) { diff --git a/sdk/Domain/UploadObjectRequest.cs b/sdk/Domain/UploadObjectRequest.cs index 6f64947..1e7aa03 100644 --- a/sdk/Domain/UploadObjectRequest.cs +++ b/sdk/Domain/UploadObjectRequest.cs @@ -17,7 +17,7 @@ namespace Aliyun.OSS /// public class UploadObjectRequest { - private int _parallelThreadCount = 2; + private int _parallelThreadCount = 3; /// /// Initializes a new instance of the class. From bb27fd975015bc5a02a939c4c4d290026c22e04f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=B8=A3=E9=95=9D?= Date: Thu, 21 Dec 2017 11:12:21 +0800 Subject: [PATCH 5/8] Refine the sample of ResumableUploadObject --- samples/Samples/ResumableSample.cs | 43 +++++++++++++++++------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/samples/Samples/ResumableSample.cs b/samples/Samples/ResumableSample.cs index 21da4f1..e670106 100644 --- a/samples/Samples/ResumableSample.cs +++ b/samples/Samples/ResumableSample.cs @@ -25,13 +25,19 @@ public static class ResumbaleSample static string fileToUpload = Config.BigFileToUpload; - public static void ResumableUploadObject(string bucketName) + public static void ResumableUploadObject(string bucketName) { const string key = "ResumableUploadObject"; string checkpointDir = Config.DirToDownload; try { - client.ResumableUploadObject(bucketName, key, fileToUpload, null, checkpointDir); + UploadObjectRequest request = new UploadObjectRequest(bucketName, key, fileToUpload) + { + PartSize = 8 * 1024 * 1024, + ParallelThreadCount = 3, + CheckpointDir = checkpointDir, + }; + client.ResumableUploadObject(request); Console.WriteLine("Resumable upload object:{0} succeeded", key); } catch (OssException ex) @@ -45,16 +51,21 @@ public static void ResumableUploadObject(string bucketName) } } - public static void ResumableCopyObject(string sourceBucketName, string sourceKey, - string destBucketName, string destKey) + public static void ResumableDownloadObject(string bucketName) { - + const string key = "ResumableDownloadObject"; + string fileToDownload = Config.DirToDownload + key; string checkpointDir = Config.DirToDownload; try { - var request = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey); - client.ResumableCopyObject(request, checkpointDir); - Console.WriteLine("Resumable copy new object:{0} succeeded", request.DestinationKey); + DownloadObjectRequest request = new DownloadObjectRequest(bucketName, key, fileToDownload) + { + PartSize = 8 * 1024 * 1024, + ParallelThreadCount = 3, + CheckpointDir = Config.DirToDownload, + }; + client.ResumableDownloadObject(request); + Console.WriteLine("Resumable download object:{0} succeeded", key); } catch (OssException ex) { @@ -67,21 +78,15 @@ public static void ResumableCopyObject(string sourceBucketName, string sourceKey } } - public static void ResumableDownloadObject(string bucketName) + public static void ResumableCopyObject(string sourceBucketName, string sourceKey, + string destBucketName, string destKey) { - const string key = "ResumableDownloadObject"; - string fileToDownload = Config.DirToDownload + key; string checkpointDir = Config.DirToDownload; try { - DownloadObjectRequest request = new DownloadObjectRequest(bucketName, key, fileToDownload) - { - PartSize = 8 * 1024 * 1024, - ParallelThreadCount = 3, - CheckpointDir = Config.DirToDownload, - }; - client.ResumableDownloadObject(request); - Console.WriteLine("Resumable download object:{0} succeeded", key); + var request = new CopyObjectRequest(sourceBucketName, sourceKey, destBucketName, destKey); + client.ResumableCopyObject(request, checkpointDir); + Console.WriteLine("Resumable copy new object:{0} succeeded", request.DestinationKey); } catch (OssException ex) { From 5c8813eade845515c2b9d3bc4c16efd3db7819c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=B8=A3=E9=95=9D?= Date: Thu, 21 Dec 2017 11:30:02 +0800 Subject: [PATCH 6/8] Release 2.7.0 --- CHANGELOG.md | 7 +++++++ README-CN.md | 2 +- README.md | 2 +- samples/Properties/AssemblyInfo.cs | 4 ++-- sdk/Properties/AssemblyInfo.cs | 4 ++-- test/Properties/AssemblyInfo.cs | 4 ++-- 6 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff21d51..55bd380 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # ChangeLog - Aliyun OSS SDK for C# +## 版本号：2.7.0 日期：2017/12/21 +### 变更内容 +- 修复：优化异步机制实现 +- 增加：支持断点并发上传下载 +- 增加：支持符号链接symlink +- 增加：LifecycleRule支持碎片及所有类型的存储 + ## 版本号：2.6.0 日期：2017/11/30 ### 变更内容 - 增加：CreateBucket支持StorageClass diff --git a/README-CN.md b/README-CN.md index bb09f34..40fec6a 100644 --- a/README-CN.md +++ b/README-CN.md @@ -12,7 +12,7 @@ - OSS C# SDK[在线文档](http://gosspublic.alicdn.com/AliyunNetSDK/apidocs/latest/index.html)。 ## 版本 - - 当前版本：2.6.0 + - 当前版本：2.7.0 ## 运行环境 diff --git a/README.md b/README.md index a0dc30d..3c4e903 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ - OSS C# SDK[Online Documentation](http://gosspublic.alicdn.com/AliyunNetSDK/international/apidocs/latest/index.html). ## Version - - Current version: 2.6.0. + - Current version: 2.7.0. ## Run environment diff --git a/samples/Properties/AssemblyInfo.cs b/samples/Properties/AssemblyInfo.cs index 0799f6b..0ddda4f 100644 --- a/samples/Properties/AssemblyInfo.cs +++ b/samples/Properties/AssemblyInfo.cs @@ -28,7 +28,7 @@ // // You can specify all the values or you can use the default the Revision and // Build Numbers by using the '*' as shown below: -[assembly: AssemblyVersion("2.6.0")] +[assembly: AssemblyVersion("2.7.0")] [assembly: NeutralResourcesLanguage("zh-CN")] -[assembly: AssemblyFileVersion("2.6.0")] +[assembly: AssemblyFileVersion("2.7.0")] diff --git a/sdk/Properties/AssemblyInfo.cs b/sdk/Properties/AssemblyInfo.cs index f1603cc..dcff8ac 100644 --- a/sdk/Properties/AssemblyInfo.cs +++ b/sdk/Properties/AssemblyInfo.cs @@ -31,11 +31,11 @@ // // Major.Minor.Build.Revision // -[assembly: AssemblyVersion("2.6.0")] +[assembly: AssemblyVersion("2.7.0")] // The asembly is designed as CLS compliant to support CLS-compliant languages. //[assembly: CLSCompliant(true)] [assembly: InternalsVisibleTo("Aliyun.OSS.UnitTest, PublicKey=002400000480000094000000060200000024000052534131000400000100010045C2B8CBBFE7B414DEE24D990688805C04B57ABB8292CEC3CFBCF4C7F6BD8254C8DDEA76F8EA035D106914678AAE9DB8BA4BF1669637043DBE62E1DE2B978729CF6F3DD0080AC2209559371D26219B50309EFDA1D51800DE052B0A45C7C9238884EEA4E7DC3C595B4930785A33A90ED4A6869285C3C04AD95245C0DFC00D24CC")] -[assembly: AssemblyFileVersion("2.6.0")] +[assembly: AssemblyFileVersion("2.7.0")] diff --git a/test/Properties/AssemblyInfo.cs b/test/Properties/AssemblyInfo.cs index d29b0fb..84d8279 100644 --- a/test/Properties/AssemblyInfo.cs +++ b/test/Properties/AssemblyInfo.cs @@ -32,7 +32,7 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2.6.0")] -[assembly: AssemblyFileVersion("2.6.0")] +[assembly: AssemblyVersion("2.7.0")] +[assembly: AssemblyFileVersion("2.7.0")] [assembly: NeutralResourcesLanguage("zh-CN")] From 84c5fb4704b941ca9636494f8b271c38514ff2fe Mon Sep 17 00:00:00 2001 From: "qi.xu" Date: Wed, 20 Dec 2017 20:32:36 -0800 Subject: [PATCH 7/8] add CRC support cross all OSS APIs Add CRC support cross all OSS APIs, including append/get/put object or part, resumable upload/download object. In resumable download API, after the downloading all the parts, it will check the CRC of the whole object. In resumable upload API, each upload part will be checked with CRC. And after the Complete API is called, the whole object CRC will be verified again. --- sdk/Commands/AppendObjectCommand.cs | 6 + sdk/Commands/PutObjectCommand.cs | 7 + sdk/Commands/UploadPartCommand.cs | 6 + sdk/Common/ClientConfiguration.cs | 12 ++ sdk/Common/Handlers/Crc64CheckHandler.cs | 48 ++++++ sdk/Common/Internal/Crc64HashAlgorithm.cs | 36 ++++ sdk/Common/Internal/HashStream.cs | 7 + sdk/Common/Internal/HashingWrapper.cs | 15 ++ sdk/Common/ResumableDownloadManager.cs | 15 ++ sdk/Common/ResumableUploadManager.cs | 9 + sdk/Domain/ObjectMetadata.cs | 20 +++ sdk/Domain/ResumableContext.cs | 36 +++- sdk/OssClient.cs | 45 ++++- sdk/Properties/AssemblyInfo.cs | 4 +- .../GetObjectResponseDeserializer.cs | 15 ++ sdk/Util/Crc64.cs | 158 ++++++++++++++++++ sdk/Util/OssUtils.cs | 38 +++++ sdk/aliyun-oss-sdk.csproj | 3 + .../ObjectBasicOperationTest.cs | 16 +- test/TestCase/ObjectTestCase/OssCrcTest.cs | 45 +++++ .../OtherTestCase/ResumableContextTest.cs | 84 ++++++++++ test/aliyun-oss-sdk-test.csproj | 2 + 22 files changed, 611 insertions(+), 16 deletions(-) create mode 100644 sdk/Common/Handlers/Crc64CheckHandler.cs create mode 100644 sdk/Common/Internal/Crc64HashAlgorithm.cs create mode 100644 sdk/Util/Crc64.cs create mode 100644 test/TestCase/ObjectTestCase/OssCrcTest.cs create mode 100644 test/TestCase/OtherTestCase/ResumableContextTest.cs diff --git a/sdk/Commands/AppendObjectCommand.cs b/sdk/Commands/AppendObjectCommand.cs index a92a94b..0da56e1 100644 --- a/sdk/Commands/AppendObjectCommand.cs +++ b/sdk/Commands/AppendObjectCommand.cs @@ -109,6 +109,12 @@ public static AppendObjectCommand Create(IServiceClient client, Uri endpoint, Ex request.Content = hashStream; context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream)); } + else if (conf.EnableCrcCheck) + { + var hashStream = new Crc64Stream(originalStream, null, streamLength); + request.Content = hashStream; + context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream)); + } return new AppendObjectCommand(client, endpoint, context, DeserializerFactory.GetFactory().CreateAppendObjectReusltDeserializer(), diff --git a/sdk/Commands/PutObjectCommand.cs b/sdk/Commands/PutObjectCommand.cs index 63986f9..ee3df99 100644 --- a/sdk/Commands/PutObjectCommand.cs +++ b/sdk/Commands/PutObjectCommand.cs @@ -116,6 +116,13 @@ public static PutObjectCommand Create(IServiceClient client, Uri endpoint, putObjectRequest.Content = hashStream; context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream)); } + else if (conf.EnableCrcCheck) + { + var streamLength = originalStream.CanSeek ? originalStream.Length : -1; + var hashStream = new Crc64Stream(originalStream, null, streamLength); + putObjectRequest.Content = hashStream; + context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream)); + } return new PutObjectCommand(client, endpoint, context, DeserializerFactory.GetFactory().CreatePutObjectReusltDeserializer(putObjectRequest), diff --git a/sdk/Commands/UploadPartCommand.cs b/sdk/Commands/UploadPartCommand.cs index bf137fd..176e2b6 100644 --- a/sdk/Commands/UploadPartCommand.cs +++ b/sdk/Commands/UploadPartCommand.cs @@ -116,6 +116,12 @@ public static UploadPartCommand Create(IServiceClient client, Uri endpoint, Exec uploadPartRequest.InputStream = hashStream; context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream)); } + else if (conf.EnableCrcCheck) + { + var hashStream = new Crc64Stream(originalStream, null, streamLength); + uploadPartRequest.InputStream = hashStream; + context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream)); + } return new UploadPartCommand(client, endpoint, context, DeserializerFactory.GetFactory().CreateUploadPartResultDeserializer(uploadPartRequest.PartNumber.Value), diff --git a/sdk/Common/ClientConfiguration.cs b/sdk/Common/ClientConfiguration.cs index 85df04f..abb7b6a 100644 --- a/sdk/Common/ClientConfiguration.cs +++ b/sdk/Common/ClientConfiguration.cs @@ -32,6 +32,7 @@ public class ClientConfiguration private long _maxPartCachingSize = 1024 * 1024 * 100; private int _preReadBufferCount = 8; private bool _useSingleThreadReadInResumableUpload = false; + private bool _enableCrcCheck = true; /// /// Max Http connection connection count. By default it's 512. @@ -138,6 +139,17 @@ public bool EnalbeMD5Check set { _enalbeMD5Check = value; } } + /// + /// Gets or sets a value indicating whether this enable + /// crc check. + /// + /// true if enable crc check; otherwise, false. + public bool EnableCrcCheck + { + get { return _enableCrcCheck; } + set { _enableCrcCheck = value; } + } + /// /// Sets the custom base time /// diff --git a/sdk/Common/Handlers/Crc64CheckHandler.cs b/sdk/Common/Handlers/Crc64CheckHandler.cs new file mode 100644 index 0000000..847e535 --- /dev/null +++ b/sdk/Common/Handlers/Crc64CheckHandler.cs @@ -0,0 +1,48 @@ +/* + * Copyright (C) Alibaba Cloud Computing + * All rights reserved. + * + */ + +using System.IO; +using Aliyun.OSS.Common.Communication; +using Aliyun.OSS.Common.Internal; +using Aliyun.OSS.Util; +using System; + +namespace Aliyun.OSS.Common.Handlers +{ + internal class Crc64CheckHandler : ResponseHandler + { + private Stream _inputStream; + + public Crc64CheckHandler(Stream inputStream) + { + _inputStream = inputStream; + } + + public override void Handle(ServiceResponse response) + { + if (_inputStream is Crc64Stream) + { + Crc64Stream stream = (Crc64Stream)_inputStream; + + if (stream.CalculatedHash == null) + { + stream.CalculateHash(); + } + if (response.Headers.ContainsKey(HttpHeaders.HashCrc64Ecma) && stream.CalculatedHash != null) + { + var sdkCalculatedHash = BitConverter.ToUInt64(stream.CalculatedHash, 0); + var ossCalculatedHashStr = response.Headers[HttpHeaders.HashCrc64Ecma]; + if (!sdkCalculatedHash.ToString().Equals(ossCalculatedHashStr)) + { + response.Dispose(); + throw new ClientException("Expected hash not equal to calculated hash"); + } + } + } + } + } +} + diff --git a/sdk/Common/Internal/Crc64HashAlgorithm.cs b/sdk/Common/Internal/Crc64HashAlgorithm.cs new file mode 100644 index 0000000..b817974 --- /dev/null +++ b/sdk/Common/Internal/Crc64HashAlgorithm.cs @@ -0,0 +1,36 @@ +using System; +using Aliyun.OSS.Util; +namespace Aliyun.OSS.Common.Internal +{ + public class Crc64HashAlgorithm : System.Security.Cryptography.HashAlgorithm + { + private ulong crc64 = 0; + + public Crc64HashAlgorithm() + { + Crc64.InitECMA(); + } + + public override void Initialize() + { + Crc64.InitECMA(); + } + + public override int HashSize + { + get { + return 64; + } + } + + protected override void HashCore(byte[] array, int ibStart, int cbSize) + { + crc64 = Crc64.Compute(array, ibStart, cbSize, crc64); + } + + protected override byte[] HashFinal() + { + return BitConverter.GetBytes(crc64); + } + } +} diff --git a/sdk/Common/Internal/HashStream.cs b/sdk/Common/Internal/HashStream.cs index 7e4c5bf..0d5131f 100644 --- a/sdk/Common/Internal/HashStream.cs +++ b/sdk/Common/Internal/HashStream.cs @@ -410,4 +410,11 @@ public MD5Stream(Stream baseStream, byte[] expectedHash, long expectedLength) #endregion } + + public class Crc64Stream : HashStream + { + public Crc64Stream(Stream baseStream, byte[] expectedHash, long expectedLength) + : base(baseStream, expectedHash, expectedLength) + { } + } } diff --git a/sdk/Common/Internal/HashingWrapper.cs b/sdk/Common/Internal/HashingWrapper.cs index ed7d80e..609ad18 100644 --- a/sdk/Common/Internal/HashingWrapper.cs +++ b/sdk/Common/Internal/HashingWrapper.cs @@ -21,9 +21,17 @@ public class HashingWrapper : IHashingWrapper private void Init(string algorithmName) { if (string.Equals(MD5ManagedName, algorithmName, StringComparison.Ordinal)) + { _algorithm = new MD5Managed(); + } + else if (string.Equals(typeof(Crc64HashAlgorithm).FullName, algorithmName, StringComparison.Ordinal)) + { + _algorithm = new Crc64HashAlgorithm(); + } else + { throw new ArgumentOutOfRangeException(algorithmName, "Unsupported hashing algorithm"); + } } public HashingWrapper(string algorithmName) @@ -109,4 +117,11 @@ public HashingWrapperMD5() : base(typeof(MD5Managed).FullName) { } } + + public class HashingWrapperCrc64 : HashingWrapper + { + public HashingWrapperCrc64() + : base(typeof(Crc64HashAlgorithm).FullName) + { } + } } diff --git a/sdk/Common/ResumableDownloadManager.cs b/sdk/Common/ResumableDownloadManager.cs index 2ca44b7..8fd732a 100644 --- a/sdk/Common/ResumableDownloadManager.cs +++ b/sdk/Common/ResumableDownloadManager.cs @@ -312,6 +312,21 @@ private void Validate(DownloadObjectRequest request, ResumableDownloadContext re } } } + else if (_conf.EnableCrcCheck && !string.IsNullOrEmpty(resumableContext.Crc64)) + { + using (var fs = File.Open(GetTempDownloadFile(request), FileMode.Open)) + { + string calcuatedCrc64 = OssUtils.ComputeContentCrc64(fs, fs.Length); + if (calcuatedCrc64 != resumableContext.Crc64) + { + throw new OssException(string.Format("The Crc64 of the downloaded file {0} does not match the expected. Expected:{1}, actual:{2}", + GetTempDownloadFile(request), + resumableContext.Crc64, + calcuatedCrc64 + )); + } + } + } File.Move(GetTempDownloadFile(request), request.DownloadFile); } diff --git a/sdk/Common/ResumableUploadManager.cs b/sdk/Common/ResumableUploadManager.cs index 51f38f7..7817cf5 100644 --- a/sdk/Common/ResumableUploadManager.cs +++ b/sdk/Common/ResumableUploadManager.cs @@ -121,6 +121,10 @@ private void DoResumableUploadSingleThread(string bucketName, string key, Resuma var partResult = _ossClient.UploadPart(request); part.PartETag = partResult.PartETag; part.IsCompleted = true; + if (partResult.ResponseMetadata.ContainsKey(HttpHeaders.HashCrc64Ecma)) + { + part.Crc64 = ulong.Parse(partResult.ResponseMetadata[HttpHeaders.HashCrc64Ecma]); + } resumableContext.Dump(); uploadedBytes += part.Length; } @@ -654,6 +658,11 @@ private void UploadPart(object state) var partResult = _ossClient.UploadPart(request); part.PartETag = partResult.PartETag; + if (partResult.ResponseMetadata.ContainsKey(HttpHeaders.HashCrc64Ecma)) + { + part.Crc64 = ulong.Parse(partResult.ResponseMetadata[HttpHeaders.HashCrc64Ecma]); + } + part.IsCompleted = true; break; } diff --git a/sdk/Domain/ObjectMetadata.cs b/sdk/Domain/ObjectMetadata.cs index 755901f..073eb1c 100644 --- a/sdk/Domain/ObjectMetadata.cs +++ b/sdk/Domain/ObjectMetadata.cs @@ -214,6 +214,26 @@ public string ContentMd5 } } + /// + /// Gets or sets the crc64. + /// + /// The crc64. + public string Crc64 + { + get{ + return _metadata.ContainsKey(HttpHeaders.HashCrc64Ecma) + ? _metadata[HttpHeaders.HashCrc64Ecma] as string : null; + } + + set + { + if (value != null) + { + _metadata[HttpHeaders.HashCrc64Ecma] = value; + } + } + } + /// /// Gets or sets the server side encryption algorithm. Only AES256 is support for now. /// diff --git a/sdk/Domain/ResumableContext.cs b/sdk/Domain/ResumableContext.cs index 0b727d6..b7a003c 100644 --- a/sdk/Domain/ResumableContext.cs +++ b/sdk/Domain/ResumableContext.cs @@ -27,6 +27,8 @@ internal class ResumablePartContext public bool IsCompleted { get; set; } + public ulong Crc64 { get; set; } + public bool FromString(string value) { if (string.IsNullOrEmpty(value)) @@ -35,7 +37,7 @@ public bool FromString(string value) } var tokens = value.Split(TokenSeparator); - if (tokens.Length != 6) + if (tokens.Length != 7) { return false; } @@ -76,6 +78,13 @@ public bool FromString(string value) PartETag = new PartETag(partNum, tokens[5]); + ulong crc = 0; + if (!ulong.TryParse(tokens[6], out crc)) + { + return false; + } + Crc64 = crc; + return true; } @@ -85,12 +94,14 @@ override public string ToString() + Length.ToString() + TokenSeparator + IsCompleted.ToString() + TokenSeparator; if (PartETag != null) { - result += PartETag.PartNumber.ToString() + TokenSeparator + PartETag.ETag; + result += PartETag.PartNumber.ToString() + TokenSeparator + PartETag.ETag + TokenSeparator; } else { result += TokenSeparator; } + result += Crc64.ToString(); + return result; } } @@ -110,6 +121,8 @@ internal class ResumableContext public string ContentMd5 { get; set; } + public string Crc64 { get; set; } + public string CheckpointFile { get @@ -153,17 +166,18 @@ public void Dump() virtual public bool FromString(string value) { var tokens = value.Split(ContextSeparator); - if (tokens.Length != 3) + if (tokens.Length != 4) { return false; } UploadId = tokens[0]; ContentMd5 = tokens[1]; - var partStr = tokens[2]; + Crc64 = tokens[2]; + var partStr = tokens[3]; var partTokens = partStr.Split(PartContextSeparator); - if (partTokens.Length <= 1) + if (partTokens.Length < 1) { return false; } @@ -191,11 +205,17 @@ override public string ToString() } result += UploadId.ToString() + ContextSeparator; - if (string.IsNullOrEmpty(ContentMd5)) + if (ContentMd5 == null) { - return string.Empty; + ContentMd5 = string.Empty; } - result += ContentMd5.ToString() + ContextSeparator; + + if (Crc64 == null) + { + Crc64 = string.Empty; + } + + result += ContentMd5 + ContextSeparator + Crc64 + ContextSeparator; if (PartContextList.Count == 0) { diff --git a/sdk/OssClient.cs b/sdk/OssClient.cs index 5aea852..410ac39 100644 --- a/sdk/OssClient.cs +++ b/sdk/OssClient.cs @@ -825,7 +825,8 @@ public PutObjectResult ResumableUploadObject(UploadObjectRequest request) } int maxRetry = ((RetryableServiceClient)_serviceClient).MaxRetryTimes; - ResumableUploadManager uploadManager = new ResumableUploadManager(this, maxRetry, OssUtils.GetClientConfiguration(_serviceClient)); + ClientConfiguration config = OssUtils.GetClientConfiguration(_serviceClient); + ResumableUploadManager uploadManager = new ResumableUploadManager(this, maxRetry, config); uploadManager.ResumableUploadWithRetry(request, resumableContext); // Completes the upload @@ -836,6 +837,8 @@ public PutObjectResult ResumableUploadObject(UploadObjectRequest request) callbackMetadata.AddHeader(HttpHeaders.Callback, metadata.HttpMetadata[HttpHeaders.Callback]); completeRequest.Metadata = callbackMetadata; } + + ulong objectCrcCalculated = 0; foreach (var part in resumableContext.PartContextList) { if (part == null || !part.IsCompleted) @@ -843,10 +846,31 @@ public PutObjectResult ResumableUploadObject(UploadObjectRequest request) throw new OssException("Not all parts are completed."); } + if (config.EnableCrcCheck) + { + if (objectCrcCalculated == 0) + { + objectCrcCalculated = part.Crc64; + } + else + { + objectCrcCalculated = Crc64.Combine(objectCrcCalculated, part.Crc64, part.Length); + } + } + completeRequest.PartETags.Add(part.PartETag); } PutObjectResult result = CompleteMultipartUpload(completeRequest); + if (config.EnableCrcCheck && result.ResponseMetadata.ContainsKey(HttpHeaders.HashCrc64Ecma)) + { + string objectCrc = result.ResponseMetadata[HttpHeaders.HashCrc64Ecma]; + if (!string.Equals(objectCrcCalculated.ToString(), objectCrc)) + { + throw new ClientException("The whole uploaded object's CRC returned from OSS does not match the calculated one. OSS returns " + + objectCrc + " but calculated is " + objectCrcCalculated); + } + } resumableContext.Clear(); return result; @@ -1030,6 +1054,15 @@ public ObjectMetadata ResumableDownloadObject(DownloadObjectRequest request) byte[] expectedHashDigest = Convert.FromBase64String(objectMeta.ContentMd5); ; streamWrapper = new MD5Stream(ossObject.Content, expectedHashDigest, fileSize); } + else if (config.EnableCrcCheck && !string.IsNullOrEmpty(objectMeta.Crc64)) + { + ulong crcVal = 0; + if (UInt64.TryParse(objectMeta.Crc64, out crcVal)) + { + byte[] expectedHashDigest = BitConverter.GetBytes(crcVal); + streamWrapper = new Crc64Stream(ossObject.Content, expectedHashDigest, fileSize); + } + } if (request.StreamTransferProgress != null) { @@ -1574,14 +1607,20 @@ private ResumableContext LoadResumableUploadContext(string bucketName, string ke private ResumableDownloadContext LoadResumableDownloadContext(string bucketName, string key, ObjectMetadata metadata, string checkpointDir, long partSize) { var resumableContext = new ResumableDownloadContext(bucketName, key, checkpointDir); - if (resumableContext.Load() && resumableContext.ETag == metadata.ETag && resumableContext.ContentMd5 == metadata.ContentMd5) + if (resumableContext.Load()) { - return resumableContext; + if (resumableContext.ETag == metadata.ETag + && resumableContext.ContentMd5 == metadata.ContentMd5 + && resumableContext.Crc64 == metadata.Crc64) + { + return resumableContext; + } } NewResumableContext(metadata.ContentLength, partSize, resumableContext); resumableContext.ContentMd5 = metadata.ContentMd5; resumableContext.ETag = metadata.ETag; + resumableContext.Crc64 = metadata.Crc64; return resumableContext; } diff --git a/sdk/Properties/AssemblyInfo.cs b/sdk/Properties/AssemblyInfo.cs index dcff8ac..3622972 100644 --- a/sdk/Properties/AssemblyInfo.cs +++ b/sdk/Properties/AssemblyInfo.cs @@ -36,6 +36,8 @@ // The asembly is designed as CLS compliant to support CLS-compliant languages. //[assembly: CLSCompliant(true)] -[assembly: InternalsVisibleTo("Aliyun.OSS.UnitTest, PublicKey=002400000480000094000000060200000024000052534131000400000100010045C2B8CBBFE7B414DEE24D990688805C04B57ABB8292CEC3CFBCF4C7F6BD8254C8DDEA76F8EA035D106914678AAE9DB8BA4BF1669637043DBE62E1DE2B978729CF6F3DD0080AC2209559371D26219B50309EFDA1D51800DE052B0A45C7C9238884EEA4E7DC3C595B4930785A33A90ED4A6869285C3C04AD95245C0DFC00D24CC")] +[assembly: InternalsVisibleTo("Aliyun.OSS.Test, PublicKey=002400000480000094000000060200000024000052534131000400000100010045C2B8CBBFE7B414DEE24D990688805C04B57ABB8292CEC3CFBCF4C7F6BD8254C8DDEA76F8EA035D106914678AAE9DB8BA4BF1669637043DBE62E1DE2B978729CF6F3DD0080AC2209559371D26219B50309EFDA1D51800DE052B0A45C7C9238884EEA4E7DC3C595B4930785A33A90ED4A6869285C3C04AD95245C0DFC00D24CC")] +[assembly: InternalsVisibleTo("Aliyun.OSS.Test")] [assembly: AssemblyFileVersion("2.7.0")] + diff --git a/sdk/Transform/GetObjectResponseDeserializer.cs b/sdk/Transform/GetObjectResponseDeserializer.cs index 36e7b58..6800305 100644 --- a/sdk/Transform/GetObjectResponseDeserializer.cs +++ b/sdk/Transform/GetObjectResponseDeserializer.cs @@ -59,6 +59,21 @@ public override OssObject Deserialize(ServiceResponse xmlStream) var hashStream = new MD5Stream(originalStream, expectedHashDigest, streamLength); ossObject.ResponseStream = hashStream; } + else if (conf.EnableCrcCheck && _getObjectRequest.Range == null) + { + byte[] expectedHashDigest = null; + if (xmlStream.Headers.ContainsKey(HttpHeaders.HashCrc64Ecma)) + { + var crcString = xmlStream.Headers[HttpHeaders.HashCrc64Ecma]; + ulong crcVal; + if (ulong.TryParse(crcString, out crcVal)) + { + expectedHashDigest = BitConverter.GetBytes(crcVal); + var hashStream = new Crc64Stream(originalStream, expectedHashDigest, streamLength); + ossObject.ResponseStream = hashStream; + } + } + } return ossObject; } diff --git a/sdk/Util/Crc64.cs b/sdk/Util/Crc64.cs new file mode 100644 index 0000000..c929e54 --- /dev/null +++ b/sdk/Util/Crc64.cs @@ -0,0 +1,158 @@ +/* + * Copyright (C) Alibaba Cloud Computing + * All rights reserved. + * + */ + +namespace Aliyun.OSS.Util +{ + public class Crc64 + { + private static ulong[] _table; + private static object _lock = new object(); + private const int GF2_DIM = 64; /* dimension of GF(2) vectors (length of CRC) */ + private static ulong _poly; + + private static void GenStdCrcTable(ulong poly) + { + _poly = poly; + + _table = new ulong[256]; + + for (uint n = 0; n < 256; n++) + { + ulong crc = n; + for (int k = 0; k < 8; k++) + { + if ((crc & 1) == 1) + { + crc = (crc >> 1) ^ poly; + } + else + { + crc = (crc >> 1); + } + } + _table[n] = crc; + } + } + + private static ulong TableValue(ulong[] table, byte b, ulong crc) + { + unchecked + { + return (crc >> 8) ^ table[(crc ^ b) & 0xffUL]; + } + } + + public static void Init(ulong poly) + { + if (_table == null) + { + lock (_lock) + { + if (_table == null) + { + GenStdCrcTable(poly); + } + } + } + } + + public static void InitECMA() + { + Init(0xC96C5795D7870F42); + } + + public static ulong Compute(byte[] bytes, int start, int size, ulong crc = 0) + { + crc = ~crc; + for (var i = start; i < start + size; i++) + { + crc = TableValue(_table, bytes[i], crc); + } + crc = ~crc; + return crc; + } + + private static ulong Gf2MatrixTimes(ulong[] mat, ulong vec) + { + ulong sum = 0; + int idx = 0; + while (vec != 0) + { + if ((vec & 1) == 1) + sum ^= mat[idx]; + vec >>= 1; + idx++; + } + return sum; + } + + private static void Gf2MatrixSquare(ulong[] square, ulong[] mat) + { + for (int n = 0; n < GF2_DIM; n++) + square[n] = Gf2MatrixTimes(mat, mat[n]); + } + + /* + * Return the CRC-64 of two sequential blocks, where summ1 is the CRC-64 of the + * first block, summ2 is the CRC-64 of the second block, and len2 is the length + * of the second block. + */ + static public ulong Combine(ulong crc1, ulong crc2, long len2) + { + // degenerate case. + if (len2 == 0) + return crc1; + + int n; + ulong row; + ulong[] even = new ulong[GF2_DIM]; // even-power-of-two zeros operator + ulong[] odd = new ulong[GF2_DIM]; // odd-power-of-two zeros operator + + // put operator for one zero bit in odd + odd[0] = _poly; // CRC-64 polynomial + + row = 1; + for (n = 1; n < GF2_DIM; n++) + { + odd[n] = row; + row <<= 1; + } + + // put operator for two zero bits in even + Gf2MatrixSquare(even, odd); + + // put operator for four zero bits in odd + Gf2MatrixSquare(odd, even); + + // apply len2 zeros to crc1 (first square will put the operator for one + // zero byte, eight zero bits, in even) + do + { + // apply zeros operator for this bit of len2 + Gf2MatrixSquare(even, odd); + if ((len2 & 1) == 1) + crc1 = Gf2MatrixTimes(even, crc1); + len2 >>= 1; + + // if no more bits set, then done + if (len2 == 0) + break; + + // another iteration of the loop with odd and even swapped + Gf2MatrixSquare(odd, even); + if ((len2 & 1) == 1) + crc1 = Gf2MatrixTimes(odd, crc1); + len2 >>= 1; + + // if no more bits set, then done + } while (len2 != 0); + + // return combined crc. + crc1 ^= crc2; + return crc1; + } + } +} diff --git a/sdk/Util/OssUtils.cs b/sdk/Util/OssUtils.cs index 52f53cc..d632955 100644 --- a/sdk/Util/OssUtils.cs +++ b/sdk/Util/OssUtils.cs @@ -230,6 +230,44 @@ public static string ComputeContentMd5(Stream input, long partSize) } } + /// + /// Computes the content crc64. + /// + /// The content crc64. + /// Input. + /// stream length + public static string ComputeContentCrc64(Stream input, long length) + { + using(Crc64Stream crcStream = new Crc64Stream(input, null, length)) + { + byte[] buffer = new byte[32 * 1024]; + int readCount = 0; + while(readCount < length) + { + int read = crcStream.Read(buffer, 0, buffer.Length); + if (read == 0) + { + break; + } + readCount += read; + } + + if (crcStream.CalculatedHash == null) + { + crcStream.CalculateHash(); + } + + if (crcStream.CalculatedHash == null || crcStream.CalculatedHash.Length == 0) + { + return string.Empty; + } + else + { + return BitConverter.ToUInt64(crcStream.CalculatedHash, 0).ToString(); + } + } + } + /// /// Checks if the webpage url is valid. /// diff --git a/sdk/aliyun-oss-sdk.csproj b/sdk/aliyun-oss-sdk.csproj index 05bfc90..8594796 100644 --- a/sdk/aliyun-oss-sdk.csproj +++ b/sdk/aliyun-oss-sdk.csproj @@ -382,6 +382,9 @@ + + + diff --git a/test/TestCase/ObjectTestCase/ObjectBasicOperationTest.cs b/test/TestCase/ObjectTestCase/ObjectBasicOperationTest.cs index 90e057c..8321f20 100644 --- a/test/TestCase/ObjectTestCase/ObjectBasicOperationTest.cs +++ b/test/TestCase/ObjectTestCase/ObjectBasicOperationTest.cs @@ -43,10 +43,18 @@ public static void ClassInitialize() _ossClient.CreateBucket(_archiveBucketName, StorageClass.Archive); //create sample object _objectKey = OssTestUtils.GetObjectKey(_className); - var poResult = OssTestUtils.UploadObject(_ossClient, _bucketName, _objectKey, - Config.UploadTestFile, new ObjectMetadata()); - _objectETag = poResult.ETag; - + try + { + var poResult = OssTestUtils.UploadObject(_ossClient, _bucketName, _objectKey, + Config.UploadTestFile, new ObjectMetadata()); + + _objectETag = poResult.ETag; + } + catch(Exception e) + { + Console.WriteLine(e.ToString()); + throw; + } _event = new AutoResetEvent(false); } diff --git a/test/TestCase/ObjectTestCase/OssCrcTest.cs b/test/TestCase/ObjectTestCase/OssCrcTest.cs new file mode 100644 index 0000000..8c34efc --- /dev/null +++ b/test/TestCase/ObjectTestCase/OssCrcTest.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Aliyun.OSS; +using Aliyun.OSS.Common; +using Aliyun.OSS.Test.Util; +using System.Text; + +using NUnit.Framework; + +using Aliyun.OSS.Util; +namespace Aliyun.OSS.Test.TestClass.ObjectTestClass +{ + [TestFixture] + public class OssCrcTest + { + [Test] + public void TestCrcCore() + { + string testStr = "This is a test."; + byte[] content = Encoding.ASCII.GetBytes(testStr); + Crc64.InitECMA(); + ulong crc = Crc64.Compute(content, 0, content.Length, 0); + Assert.AreEqual(crc, 2186167744391481992); + } + + [Test] + public void TestCombine() + { + string str1 = "this is a test."; + string str2 = "hello world."; + string str = str1 + str2; + + byte[] content1 = Encoding.ASCII.GetBytes(str1); + byte[] content2 = Encoding.ASCII.GetBytes(str2); + byte[] content = Encoding.ASCII.GetBytes(str); + + Crc64.InitECMA(); + ulong crc1 = Crc64.Compute(content1, 0, content1.Length); + ulong crc2 = Crc64.Compute(content2, 0, content2.Length); + ulong crc = Crc64.Compute(content, 0, content.Length); + Assert.AreEqual(crc, Crc64.Combine(crc1, crc2, content2.Length)); + } + } +} diff --git a/test/TestCase/OtherTestCase/ResumableContextTest.cs b/test/TestCase/OtherTestCase/ResumableContextTest.cs new file mode 100644 index 0000000..04c74df --- /dev/null +++ b/test/TestCase/OtherTestCase/ResumableContextTest.cs @@ -0,0 +1,84 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using Aliyun.OSS; +using Aliyun.OSS.Test.Util; + +using NUnit.Framework; +using System.Threading; +using System; +namespace Aliyun.OSS.Test.TestClass.OtherTestClass +{ + [TestFixture] + public class ResumableContextTest + { + [Test] + public void ResumablePartContextSerializationTest() + { + ResumablePartContext partContext = new ResumablePartContext(); + partContext.Position = 1000; + partContext.Length = 1024 * 100; + partContext.IsCompleted = true; + partContext.Crc64 = 49417943; + partContext.PartId = 1; + partContext.PartETag = new PartETag(3, "1234567890"); + + string s = partContext.ToString(); + + ResumablePartContext partContext2 = new ResumablePartContext(); + partContext2.FromString(s); + Assert.AreEqual(partContext.Position, partContext2.Position); + Assert.AreEqual(partContext.Length, partContext2.Length); + Assert.AreEqual(partContext.IsCompleted, partContext2.IsCompleted); + Assert.AreEqual(partContext.Crc64, partContext2.Crc64); + Assert.AreEqual(partContext.PartETag.ETag, partContext2.PartETag.ETag); + Assert.AreEqual(partContext.PartETag.PartNumber, partContext2.PartETag.PartNumber); + } + + [Test] + public void ResumableContextSerializationTest() + { + ResumableContextSerializationTest("testMd5"); + } + + [Test] + public void ResumableContextWithoutMd5SerializationTest() + { + ResumableContextSerializationTest(null); + } + + public void ResumableContextSerializationTest(string md5) + { + ResumableContext resumableContext = new ResumableContext("bk1", "key1", "chkdir"); + resumableContext.UploadId = "UploadId"; + resumableContext.PartContextList = new List(); + ResumablePartContext partContext = new ResumablePartContext(); + partContext.Position = 1000; + partContext.Length = 1024 * 100; + partContext.IsCompleted = true; + partContext.Crc64 = 49417943; + partContext.PartId = 1; + partContext.PartETag = new PartETag(3, "1234567890"); + + resumableContext.PartContextList.Add(partContext); + resumableContext.ContentMd5 = md5; + resumableContext.Crc64 = "1234567890"; + + string s = resumableContext.ToString(); + + ResumableContext resumableContext2 = new ResumableContext("bk2", "key2", "chdir2"); + resumableContext2.FromString(s); + Assert.AreEqual(resumableContext.ContentMd5, resumableContext2.ContentMd5); + Assert.AreEqual(resumableContext.Crc64, resumableContext2.Crc64); + Assert.AreEqual(resumableContext.PartContextList.Count, resumableContext2.PartContextList.Count); + + ResumablePartContext partContext2 = resumableContext2.PartContextList[0]; + Assert.AreEqual(partContext.Position, partContext2.Position); + Assert.AreEqual(partContext.Length, partContext2.Length); + Assert.AreEqual(partContext.IsCompleted, partContext2.IsCompleted); + Assert.AreEqual(partContext.Crc64, partContext2.Crc64); + Assert.AreEqual(partContext.PartETag.ETag, partContext2.PartETag.ETag); + Assert.AreEqual(partContext.PartETag.PartNumber, partContext2.PartETag.PartNumber); + } + } +} diff --git a/test/aliyun-oss-sdk-test.csproj b/test/aliyun-oss-sdk-test.csproj index 548d904..c8ca1e5 100644 --- a/test/aliyun-oss-sdk-test.csproj +++ b/test/aliyun-oss-sdk-test.csproj @@ -74,6 +74,8 @@ + + From 9254f7e4dbb9b64a1c520e81fb7cce46a62fa178 Mon Sep 17 00:00:00 2001 From: "qi.xu" Date: Wed, 20 Dec 2017 20:42:01 -0800 Subject: [PATCH 8/8] Revert "add CRC support cross all OSS APIs" This reverts commit 84c5fb4704b941ca9636494f8b271c38514ff2fe. --- sdk/Commands/AppendObjectCommand.cs | 6 - sdk/Commands/PutObjectCommand.cs | 7 - sdk/Commands/UploadPartCommand.cs | 6 - sdk/Common/ClientConfiguration.cs | 12 -- sdk/Common/Handlers/Crc64CheckHandler.cs | 48 ------ sdk/Common/Internal/Crc64HashAlgorithm.cs | 36 ---- sdk/Common/Internal/HashStream.cs | 7 - sdk/Common/Internal/HashingWrapper.cs | 15 -- sdk/Common/ResumableDownloadManager.cs | 15 -- sdk/Common/ResumableUploadManager.cs | 9 - sdk/Domain/ObjectMetadata.cs | 20 --- sdk/Domain/ResumableContext.cs | 36 +--- sdk/OssClient.cs | 45 +---- sdk/Properties/AssemblyInfo.cs | 4 +- .../GetObjectResponseDeserializer.cs | 15 -- sdk/Util/Crc64.cs | 158 ------------------ sdk/Util/OssUtils.cs | 38 ----- sdk/aliyun-oss-sdk.csproj | 3 - .../ObjectBasicOperationTest.cs | 16 +- test/TestCase/ObjectTestCase/OssCrcTest.cs | 45 ----- .../OtherTestCase/ResumableContextTest.cs | 84 ---------- test/aliyun-oss-sdk-test.csproj | 2 - 22 files changed, 16 insertions(+), 611 deletions(-) delete mode 100644 sdk/Common/Handlers/Crc64CheckHandler.cs delete mode 100644 sdk/Common/Internal/Crc64HashAlgorithm.cs delete mode 100644 sdk/Util/Crc64.cs delete mode 100644 test/TestCase/ObjectTestCase/OssCrcTest.cs delete mode 100644 test/TestCase/OtherTestCase/ResumableContextTest.cs diff --git a/sdk/Commands/AppendObjectCommand.cs b/sdk/Commands/AppendObjectCommand.cs index 0da56e1..a92a94b 100644 --- a/sdk/Commands/AppendObjectCommand.cs +++ b/sdk/Commands/AppendObjectCommand.cs @@ -109,12 +109,6 @@ public static AppendObjectCommand Create(IServiceClient client, Uri endpoint, Ex request.Content = hashStream; context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream)); } - else if (conf.EnableCrcCheck) - { - var hashStream = new Crc64Stream(originalStream, null, streamLength); - request.Content = hashStream; - context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream)); - } return new AppendObjectCommand(client, endpoint, context, DeserializerFactory.GetFactory().CreateAppendObjectReusltDeserializer(), diff --git a/sdk/Commands/PutObjectCommand.cs b/sdk/Commands/PutObjectCommand.cs index ee3df99..63986f9 100644 --- a/sdk/Commands/PutObjectCommand.cs +++ b/sdk/Commands/PutObjectCommand.cs @@ -116,13 +116,6 @@ public static PutObjectCommand Create(IServiceClient client, Uri endpoint, putObjectRequest.Content = hashStream; context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream)); } - else if (conf.EnableCrcCheck) - { - var streamLength = originalStream.CanSeek ? originalStream.Length : -1; - var hashStream = new Crc64Stream(originalStream, null, streamLength); - putObjectRequest.Content = hashStream; - context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream)); - } return new PutObjectCommand(client, endpoint, context, DeserializerFactory.GetFactory().CreatePutObjectReusltDeserializer(putObjectRequest), diff --git a/sdk/Commands/UploadPartCommand.cs b/sdk/Commands/UploadPartCommand.cs index 176e2b6..bf137fd 100644 --- a/sdk/Commands/UploadPartCommand.cs +++ b/sdk/Commands/UploadPartCommand.cs @@ -116,12 +116,6 @@ public static UploadPartCommand Create(IServiceClient client, Uri endpoint, Exec uploadPartRequest.InputStream = hashStream; context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream)); } - else if (conf.EnableCrcCheck) - { - var hashStream = new Crc64Stream(originalStream, null, streamLength); - uploadPartRequest.InputStream = hashStream; - context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream)); - } return new UploadPartCommand(client, endpoint, context, DeserializerFactory.GetFactory().CreateUploadPartResultDeserializer(uploadPartRequest.PartNumber.Value), diff --git a/sdk/Common/ClientConfiguration.cs b/sdk/Common/ClientConfiguration.cs index abb7b6a..85df04f 100644 --- a/sdk/Common/ClientConfiguration.cs +++ b/sdk/Common/ClientConfiguration.cs @@ -32,7 +32,6 @@ public class ClientConfiguration private long _maxPartCachingSize = 1024 * 1024 * 100; private int _preReadBufferCount = 8; private bool _useSingleThreadReadInResumableUpload = false; - private bool _enableCrcCheck = true; /// /// Max Http connection connection count. By default it's 512. @@ -139,17 +138,6 @@ public bool EnalbeMD5Check set { _enalbeMD5Check = value; } } - /// - /// Gets or sets a value indicating whether this enable - /// crc check. - /// - /// true if enable crc check; otherwise, false. - public bool EnableCrcCheck - { - get { return _enableCrcCheck; } - set { _enableCrcCheck = value; } - } - /// /// Sets the custom base time /// diff --git a/sdk/Common/Handlers/Crc64CheckHandler.cs b/sdk/Common/Handlers/Crc64CheckHandler.cs deleted file mode 100644 index 847e535..0000000 --- a/sdk/Common/Handlers/Crc64CheckHandler.cs +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) Alibaba Cloud Computing - * All rights reserved. - * - */ - -using System.IO; -using Aliyun.OSS.Common.Communication; -using Aliyun.OSS.Common.Internal; -using Aliyun.OSS.Util; -using System; - -namespace Aliyun.OSS.Common.Handlers -{ - internal class Crc64CheckHandler : ResponseHandler - { - private Stream _inputStream; - - public Crc64CheckHandler(Stream inputStream) - { - _inputStream = inputStream; - } - - public override void Handle(ServiceResponse response) - { - if (_inputStream is Crc64Stream) - { - Crc64Stream stream = (Crc64Stream)_inputStream; - - if (stream.CalculatedHash == null) - { - stream.CalculateHash(); - } - if (response.Headers.ContainsKey(HttpHeaders.HashCrc64Ecma) && stream.CalculatedHash != null) - { - var sdkCalculatedHash = BitConverter.ToUInt64(stream.CalculatedHash, 0); - var ossCalculatedHashStr = response.Headers[HttpHeaders.HashCrc64Ecma]; - if (!sdkCalculatedHash.ToString().Equals(ossCalculatedHashStr)) - { - response.Dispose(); - throw new ClientException("Expected hash not equal to calculated hash"); - } - } - } - } - } -} - diff --git a/sdk/Common/Internal/Crc64HashAlgorithm.cs b/sdk/Common/Internal/Crc64HashAlgorithm.cs deleted file mode 100644 index b817974..0000000 --- a/sdk/Common/Internal/Crc64HashAlgorithm.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using Aliyun.OSS.Util; -namespace Aliyun.OSS.Common.Internal -{ - public class Crc64HashAlgorithm : System.Security.Cryptography.HashAlgorithm - { - private ulong crc64 = 0; - - public Crc64HashAlgorithm() - { - Crc64.InitECMA(); - } - - public override void Initialize() - { - Crc64.InitECMA(); - } - - public override int HashSize - { - get { - return 64; - } - } - - protected override void HashCore(byte[] array, int ibStart, int cbSize) - { - crc64 = Crc64.Compute(array, ibStart, cbSize, crc64); - } - - protected override byte[] HashFinal() - { - return BitConverter.GetBytes(crc64); - } - } -} diff --git a/sdk/Common/Internal/HashStream.cs b/sdk/Common/Internal/HashStream.cs index 0d5131f..7e4c5bf 100644 --- a/sdk/Common/Internal/HashStream.cs +++ b/sdk/Common/Internal/HashStream.cs @@ -410,11 +410,4 @@ public MD5Stream(Stream baseStream, byte[] expectedHash, long expectedLength) #endregion } - - public class Crc64Stream : HashStream - { - public Crc64Stream(Stream baseStream, byte[] expectedHash, long expectedLength) - : base(baseStream, expectedHash, expectedLength) - { } - } } diff --git a/sdk/Common/Internal/HashingWrapper.cs b/sdk/Common/Internal/HashingWrapper.cs index 609ad18..ed7d80e 100644 --- a/sdk/Common/Internal/HashingWrapper.cs +++ b/sdk/Common/Internal/HashingWrapper.cs @@ -21,17 +21,9 @@ public class HashingWrapper : IHashingWrapper private void Init(string algorithmName) { if (string.Equals(MD5ManagedName, algorithmName, StringComparison.Ordinal)) - { _algorithm = new MD5Managed(); - } - else if (string.Equals(typeof(Crc64HashAlgorithm).FullName, algorithmName, StringComparison.Ordinal)) - { - _algorithm = new Crc64HashAlgorithm(); - } else - { throw new ArgumentOutOfRangeException(algorithmName, "Unsupported hashing algorithm"); - } } public HashingWrapper(string algorithmName) @@ -117,11 +109,4 @@ public HashingWrapperMD5() : base(typeof(MD5Managed).FullName) { } } - - public class HashingWrapperCrc64 : HashingWrapper - { - public HashingWrapperCrc64() - : base(typeof(Crc64HashAlgorithm).FullName) - { } - } } diff --git a/sdk/Common/ResumableDownloadManager.cs b/sdk/Common/ResumableDownloadManager.cs index 8fd732a..2ca44b7 100644 --- a/sdk/Common/ResumableDownloadManager.cs +++ b/sdk/Common/ResumableDownloadManager.cs @@ -312,21 +312,6 @@ private void Validate(DownloadObjectRequest request, ResumableDownloadContext re } } } - else if (_conf.EnableCrcCheck && !string.IsNullOrEmpty(resumableContext.Crc64)) - { - using (var fs = File.Open(GetTempDownloadFile(request), FileMode.Open)) - { - string calcuatedCrc64 = OssUtils.ComputeContentCrc64(fs, fs.Length); - if (calcuatedCrc64 != resumableContext.Crc64) - { - throw new OssException(string.Format("The Crc64 of the downloaded file {0} does not match the expected. Expected:{1}, actual:{2}", - GetTempDownloadFile(request), - resumableContext.Crc64, - calcuatedCrc64 - )); - } - } - } File.Move(GetTempDownloadFile(request), request.DownloadFile); } diff --git a/sdk/Common/ResumableUploadManager.cs b/sdk/Common/ResumableUploadManager.cs index 7817cf5..51f38f7 100644 --- a/sdk/Common/ResumableUploadManager.cs +++ b/sdk/Common/ResumableUploadManager.cs @@ -121,10 +121,6 @@ private void DoResumableUploadSingleThread(string bucketName, string key, Resuma var partResult = _ossClient.UploadPart(request); part.PartETag = partResult.PartETag; part.IsCompleted = true; - if (partResult.ResponseMetadata.ContainsKey(HttpHeaders.HashCrc64Ecma)) - { - part.Crc64 = ulong.Parse(partResult.ResponseMetadata[HttpHeaders.HashCrc64Ecma]); - } resumableContext.Dump(); uploadedBytes += part.Length; } @@ -658,11 +654,6 @@ private void UploadPart(object state) var partResult = _ossClient.UploadPart(request); part.PartETag = partResult.PartETag; - if (partResult.ResponseMetadata.ContainsKey(HttpHeaders.HashCrc64Ecma)) - { - part.Crc64 = ulong.Parse(partResult.ResponseMetadata[HttpHeaders.HashCrc64Ecma]); - } - part.IsCompleted = true; break; } diff --git a/sdk/Domain/ObjectMetadata.cs b/sdk/Domain/ObjectMetadata.cs index 073eb1c..755901f 100644 --- a/sdk/Domain/ObjectMetadata.cs +++ b/sdk/Domain/ObjectMetadata.cs @@ -214,26 +214,6 @@ public string ContentMd5 } } - /// - /// Gets or sets the crc64. - /// - /// The crc64. - public string Crc64 - { - get{ - return _metadata.ContainsKey(HttpHeaders.HashCrc64Ecma) - ? _metadata[HttpHeaders.HashCrc64Ecma] as string : null; - } - - set - { - if (value != null) - { - _metadata[HttpHeaders.HashCrc64Ecma] = value; - } - } - } - /// /// Gets or sets the server side encryption algorithm. Only AES256 is support for now. /// diff --git a/sdk/Domain/ResumableContext.cs b/sdk/Domain/ResumableContext.cs index b7a003c..0b727d6 100644 --- a/sdk/Domain/ResumableContext.cs +++ b/sdk/Domain/ResumableContext.cs @@ -27,8 +27,6 @@ internal class ResumablePartContext public bool IsCompleted { get; set; } - public ulong Crc64 { get; set; } - public bool FromString(string value) { if (string.IsNullOrEmpty(value)) @@ -37,7 +35,7 @@ public bool FromString(string value) } var tokens = value.Split(TokenSeparator); - if (tokens.Length != 7) + if (tokens.Length != 6) { return false; } @@ -78,13 +76,6 @@ public bool FromString(string value) PartETag = new PartETag(partNum, tokens[5]); - ulong crc = 0; - if (!ulong.TryParse(tokens[6], out crc)) - { - return false; - } - Crc64 = crc; - return true; } @@ -94,14 +85,12 @@ override public string ToString() + Length.ToString() + TokenSeparator + IsCompleted.ToString() + TokenSeparator; if (PartETag != null) { - result += PartETag.PartNumber.ToString() + TokenSeparator + PartETag.ETag + TokenSeparator; + result += PartETag.PartNumber.ToString() + TokenSeparator + PartETag.ETag; } else { result += TokenSeparator; } - result += Crc64.ToString(); - return result; } } @@ -121,8 +110,6 @@ internal class ResumableContext public string ContentMd5 { get; set; } - public string Crc64 { get; set; } - public string CheckpointFile { get @@ -166,18 +153,17 @@ public void Dump() virtual public bool FromString(string value) { var tokens = value.Split(ContextSeparator); - if (tokens.Length != 4) + if (tokens.Length != 3) { return false; } UploadId = tokens[0]; ContentMd5 = tokens[1]; - Crc64 = tokens[2]; - var partStr = tokens[3]; + var partStr = tokens[2]; var partTokens = partStr.Split(PartContextSeparator); - if (partTokens.Length < 1) + if (partTokens.Length <= 1) { return false; } @@ -205,17 +191,11 @@ override public string ToString() } result += UploadId.ToString() + ContextSeparator; - if (ContentMd5 == null) + if (string.IsNullOrEmpty(ContentMd5)) { - ContentMd5 = string.Empty; - } - - if (Crc64 == null) - { - Crc64 = string.Empty; + return string.Empty; } - - result += ContentMd5 + ContextSeparator + Crc64 + ContextSeparator; + result += ContentMd5.ToString() + ContextSeparator; if (PartContextList.Count == 0) { diff --git a/sdk/OssClient.cs b/sdk/OssClient.cs index 410ac39..5aea852 100644 --- a/sdk/OssClient.cs +++ b/sdk/OssClient.cs @@ -825,8 +825,7 @@ public PutObjectResult ResumableUploadObject(UploadObjectRequest request) } int maxRetry = ((RetryableServiceClient)_serviceClient).MaxRetryTimes; - ClientConfiguration config = OssUtils.GetClientConfiguration(_serviceClient); - ResumableUploadManager uploadManager = new ResumableUploadManager(this, maxRetry, config); + ResumableUploadManager uploadManager = new ResumableUploadManager(this, maxRetry, OssUtils.GetClientConfiguration(_serviceClient)); uploadManager.ResumableUploadWithRetry(request, resumableContext); // Completes the upload @@ -837,8 +836,6 @@ public PutObjectResult ResumableUploadObject(UploadObjectRequest request) callbackMetadata.AddHeader(HttpHeaders.Callback, metadata.HttpMetadata[HttpHeaders.Callback]); completeRequest.Metadata = callbackMetadata; } - - ulong objectCrcCalculated = 0; foreach (var part in resumableContext.PartContextList) { if (part == null || !part.IsCompleted) @@ -846,31 +843,10 @@ public PutObjectResult ResumableUploadObject(UploadObjectRequest request) throw new OssException("Not all parts are completed."); } - if (config.EnableCrcCheck) - { - if (objectCrcCalculated == 0) - { - objectCrcCalculated = part.Crc64; - } - else - { - objectCrcCalculated = Crc64.Combine(objectCrcCalculated, part.Crc64, part.Length); - } - } - completeRequest.PartETags.Add(part.PartETag); } PutObjectResult result = CompleteMultipartUpload(completeRequest); - if (config.EnableCrcCheck && result.ResponseMetadata.ContainsKey(HttpHeaders.HashCrc64Ecma)) - { - string objectCrc = result.ResponseMetadata[HttpHeaders.HashCrc64Ecma]; - if (!string.Equals(objectCrcCalculated.ToString(), objectCrc)) - { - throw new ClientException("The whole uploaded object's CRC returned from OSS does not match the calculated one. OSS returns " - + objectCrc + " but calculated is " + objectCrcCalculated); - } - } resumableContext.Clear(); return result; @@ -1054,15 +1030,6 @@ public ObjectMetadata ResumableDownloadObject(DownloadObjectRequest request) byte[] expectedHashDigest = Convert.FromBase64String(objectMeta.ContentMd5); ; streamWrapper = new MD5Stream(ossObject.Content, expectedHashDigest, fileSize); } - else if (config.EnableCrcCheck && !string.IsNullOrEmpty(objectMeta.Crc64)) - { - ulong crcVal = 0; - if (UInt64.TryParse(objectMeta.Crc64, out crcVal)) - { - byte[] expectedHashDigest = BitConverter.GetBytes(crcVal); - streamWrapper = new Crc64Stream(ossObject.Content, expectedHashDigest, fileSize); - } - } if (request.StreamTransferProgress != null) { @@ -1607,20 +1574,14 @@ private ResumableContext LoadResumableUploadContext(string bucketName, string ke private ResumableDownloadContext LoadResumableDownloadContext(string bucketName, string key, ObjectMetadata metadata, string checkpointDir, long partSize) { var resumableContext = new ResumableDownloadContext(bucketName, key, checkpointDir); - if (resumableContext.Load()) + if (resumableContext.Load() && resumableContext.ETag == metadata.ETag && resumableContext.ContentMd5 == metadata.ContentMd5) { - if (resumableContext.ETag == metadata.ETag - && resumableContext.ContentMd5 == metadata.ContentMd5 - && resumableContext.Crc64 == metadata.Crc64) - { - return resumableContext; - } + return resumableContext; } NewResumableContext(metadata.ContentLength, partSize, resumableContext); resumableContext.ContentMd5 = metadata.ContentMd5; resumableContext.ETag = metadata.ETag; - resumableContext.Crc64 = metadata.Crc64; return resumableContext; } diff --git a/sdk/Properties/AssemblyInfo.cs b/sdk/Properties/AssemblyInfo.cs index 3622972..dcff8ac 100644 --- a/sdk/Properties/AssemblyInfo.cs +++ b/sdk/Properties/AssemblyInfo.cs @@ -36,8 +36,6 @@ // The asembly is designed as CLS compliant to support CLS-compliant languages. //[assembly: CLSCompliant(true)] -[assembly: InternalsVisibleTo("Aliyun.OSS.Test, PublicKey=002400000480000094000000060200000024000052534131000400000100010045C2B8CBBFE7B414DEE24D990688805C04B57ABB8292CEC3CFBCF4C7F6BD8254C8DDEA76F8EA035D106914678AAE9DB8BA4BF1669637043DBE62E1DE2B978729CF6F3DD0080AC2209559371D26219B50309EFDA1D51800DE052B0A45C7C9238884EEA4E7DC3C595B4930785A33A90ED4A6869285C3C04AD95245C0DFC00D24CC")] -[assembly: InternalsVisibleTo("Aliyun.OSS.Test")] +[assembly: InternalsVisibleTo("Aliyun.OSS.UnitTest, PublicKey=002400000480000094000000060200000024000052534131000400000100010045C2B8CBBFE7B414DEE24D990688805C04B57ABB8292CEC3CFBCF4C7F6BD8254C8DDEA76F8EA035D106914678AAE9DB8BA4BF1669637043DBE62E1DE2B978729CF6F3DD0080AC2209559371D26219B50309EFDA1D51800DE052B0A45C7C9238884EEA4E7DC3C595B4930785A33A90ED4A6869285C3C04AD95245C0DFC00D24CC")] [assembly: AssemblyFileVersion("2.7.0")] - diff --git a/sdk/Transform/GetObjectResponseDeserializer.cs b/sdk/Transform/GetObjectResponseDeserializer.cs index 6800305..36e7b58 100644 --- a/sdk/Transform/GetObjectResponseDeserializer.cs +++ b/sdk/Transform/GetObjectResponseDeserializer.cs @@ -59,21 +59,6 @@ public override OssObject Deserialize(ServiceResponse xmlStream) var hashStream = new MD5Stream(originalStream, expectedHashDigest, streamLength); ossObject.ResponseStream = hashStream; } - else if (conf.EnableCrcCheck && _getObjectRequest.Range == null) - { - byte[] expectedHashDigest = null; - if (xmlStream.Headers.ContainsKey(HttpHeaders.HashCrc64Ecma)) - { - var crcString = xmlStream.Headers[HttpHeaders.HashCrc64Ecma]; - ulong crcVal; - if (ulong.TryParse(crcString, out crcVal)) - { - expectedHashDigest = BitConverter.GetBytes(crcVal); - var hashStream = new Crc64Stream(originalStream, expectedHashDigest, streamLength); - ossObject.ResponseStream = hashStream; - } - } - } return ossObject; } diff --git a/sdk/Util/Crc64.cs b/sdk/Util/Crc64.cs deleted file mode 100644 index c929e54..0000000 --- a/sdk/Util/Crc64.cs +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (C) Alibaba Cloud Computing - * All rights reserved. - * - */ - -namespace Aliyun.OSS.Util -{ - public class Crc64 - { - private static ulong[] _table; - private static object _lock = new object(); - private const int GF2_DIM = 64; /* dimension of GF(2) vectors (length of CRC) */ - private static ulong _poly; - - private static void GenStdCrcTable(ulong poly) - { - _poly = poly; - - _table = new ulong[256]; - - for (uint n = 0; n < 256; n++) - { - ulong crc = n; - for (int k = 0; k < 8; k++) - { - if ((crc & 1) == 1) - { - crc = (crc >> 1) ^ poly; - } - else - { - crc = (crc >> 1); - } - } - _table[n] = crc; - } - } - - private static ulong TableValue(ulong[] table, byte b, ulong crc) - { - unchecked - { - return (crc >> 8) ^ table[(crc ^ b) & 0xffUL]; - } - } - - public static void Init(ulong poly) - { - if (_table == null) - { - lock (_lock) - { - if (_table == null) - { - GenStdCrcTable(poly); - } - } - } - } - - public static void InitECMA() - { - Init(0xC96C5795D7870F42); - } - - public static ulong Compute(byte[] bytes, int start, int size, ulong crc = 0) - { - crc = ~crc; - for (var i = start; i < start + size; i++) - { - crc = TableValue(_table, bytes[i], crc); - } - crc = ~crc; - return crc; - } - - private static ulong Gf2MatrixTimes(ulong[] mat, ulong vec) - { - ulong sum = 0; - int idx = 0; - while (vec != 0) - { - if ((vec & 1) == 1) - sum ^= mat[idx]; - vec >>= 1; - idx++; - } - return sum; - } - - private static void Gf2MatrixSquare(ulong[] square, ulong[] mat) - { - for (int n = 0; n < GF2_DIM; n++) - square[n] = Gf2MatrixTimes(mat, mat[n]); - } - - /* - * Return the CRC-64 of two sequential blocks, where summ1 is the CRC-64 of the - * first block, summ2 is the CRC-64 of the second block, and len2 is the length - * of the second block. - */ - static public ulong Combine(ulong crc1, ulong crc2, long len2) - { - // degenerate case. - if (len2 == 0) - return crc1; - - int n; - ulong row; - ulong[] even = new ulong[GF2_DIM]; // even-power-of-two zeros operator - ulong[] odd = new ulong[GF2_DIM]; // odd-power-of-two zeros operator - - // put operator for one zero bit in odd - odd[0] = _poly; // CRC-64 polynomial - - row = 1; - for (n = 1; n < GF2_DIM; n++) - { - odd[n] = row; - row <<= 1; - } - - // put operator for two zero bits in even - Gf2MatrixSquare(even, odd); - - // put operator for four zero bits in odd - Gf2MatrixSquare(odd, even); - - // apply len2 zeros to crc1 (first square will put the operator for one - // zero byte, eight zero bits, in even) - do - { - // apply zeros operator for this bit of len2 - Gf2MatrixSquare(even, odd); - if ((len2 & 1) == 1) - crc1 = Gf2MatrixTimes(even, crc1); - len2 >>= 1; - - // if no more bits set, then done - if (len2 == 0) - break; - - // another iteration of the loop with odd and even swapped - Gf2MatrixSquare(odd, even); - if ((len2 & 1) == 1) - crc1 = Gf2MatrixTimes(odd, crc1); - len2 >>= 1; - - // if no more bits set, then done - } while (len2 != 0); - - // return combined crc. - crc1 ^= crc2; - return crc1; - } - } -} diff --git a/sdk/Util/OssUtils.cs b/sdk/Util/OssUtils.cs index d632955..52f53cc 100644 --- a/sdk/Util/OssUtils.cs +++ b/sdk/Util/OssUtils.cs @@ -230,44 +230,6 @@ public static string ComputeContentMd5(Stream input, long partSize) } } - /// - /// Computes the content crc64. - /// - /// The content crc64. - /// Input. - /// stream length - public static string ComputeContentCrc64(Stream input, long length) - { - using(Crc64Stream crcStream = new Crc64Stream(input, null, length)) - { - byte[] buffer = new byte[32 * 1024]; - int readCount = 0; - while(readCount < length) - { - int read = crcStream.Read(buffer, 0, buffer.Length); - if (read == 0) - { - break; - } - readCount += read; - } - - if (crcStream.CalculatedHash == null) - { - crcStream.CalculateHash(); - } - - if (crcStream.CalculatedHash == null || crcStream.CalculatedHash.Length == 0) - { - return string.Empty; - } - else - { - return BitConverter.ToUInt64(crcStream.CalculatedHash, 0).ToString(); - } - } - } - /// /// Checks if the webpage url is valid. /// diff --git a/sdk/aliyun-oss-sdk.csproj b/sdk/aliyun-oss-sdk.csproj index 8594796..05bfc90 100644 --- a/sdk/aliyun-oss-sdk.csproj +++ b/sdk/aliyun-oss-sdk.csproj @@ -382,9 +382,6 @@ - - - diff --git a/test/TestCase/ObjectTestCase/ObjectBasicOperationTest.cs b/test/TestCase/ObjectTestCase/ObjectBasicOperationTest.cs index 8321f20..90e057c 100644 --- a/test/TestCase/ObjectTestCase/ObjectBasicOperationTest.cs +++ b/test/TestCase/ObjectTestCase/ObjectBasicOperationTest.cs @@ -43,18 +43,10 @@ public static void ClassInitialize() _ossClient.CreateBucket(_archiveBucketName, StorageClass.Archive); //create sample object _objectKey = OssTestUtils.GetObjectKey(_className); - try - { - var poResult = OssTestUtils.UploadObject(_ossClient, _bucketName, _objectKey, - Config.UploadTestFile, new ObjectMetadata()); - - _objectETag = poResult.ETag; - } - catch(Exception e) - { - Console.WriteLine(e.ToString()); - throw; - } + var poResult = OssTestUtils.UploadObject(_ossClient, _bucketName, _objectKey, + Config.UploadTestFile, new ObjectMetadata()); + _objectETag = poResult.ETag; + _event = new AutoResetEvent(false); } diff --git a/test/TestCase/ObjectTestCase/OssCrcTest.cs b/test/TestCase/ObjectTestCase/OssCrcTest.cs deleted file mode 100644 index 8c34efc..0000000 --- a/test/TestCase/ObjectTestCase/OssCrcTest.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using Aliyun.OSS; -using Aliyun.OSS.Common; -using Aliyun.OSS.Test.Util; -using System.Text; - -using NUnit.Framework; - -using Aliyun.OSS.Util; -namespace Aliyun.OSS.Test.TestClass.ObjectTestClass -{ - [TestFixture] - public class OssCrcTest - { - [Test] - public void TestCrcCore() - { - string testStr = "This is a test."; - byte[] content = Encoding.ASCII.GetBytes(testStr); - Crc64.InitECMA(); - ulong crc = Crc64.Compute(content, 0, content.Length, 0); - Assert.AreEqual(crc, 2186167744391481992); - } - - [Test] - public void TestCombine() - { - string str1 = "this is a test."; - string str2 = "hello world."; - string str = str1 + str2; - - byte[] content1 = Encoding.ASCII.GetBytes(str1); - byte[] content2 = Encoding.ASCII.GetBytes(str2); - byte[] content = Encoding.ASCII.GetBytes(str); - - Crc64.InitECMA(); - ulong crc1 = Crc64.Compute(content1, 0, content1.Length); - ulong crc2 = Crc64.Compute(content2, 0, content2.Length); - ulong crc = Crc64.Compute(content, 0, content.Length); - Assert.AreEqual(crc, Crc64.Combine(crc1, crc2, content2.Length)); - } - } -} diff --git a/test/TestCase/OtherTestCase/ResumableContextTest.cs b/test/TestCase/OtherTestCase/ResumableContextTest.cs deleted file mode 100644 index 04c74df..0000000 --- a/test/TestCase/OtherTestCase/ResumableContextTest.cs +++ /dev/null @@ -1,84 +0,0 @@ -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using Aliyun.OSS; -using Aliyun.OSS.Test.Util; - -using NUnit.Framework; -using System.Threading; -using System; -namespace Aliyun.OSS.Test.TestClass.OtherTestClass -{ - [TestFixture] - public class ResumableContextTest - { - [Test] - public void ResumablePartContextSerializationTest() - { - ResumablePartContext partContext = new ResumablePartContext(); - partContext.Position = 1000; - partContext.Length = 1024 * 100; - partContext.IsCompleted = true; - partContext.Crc64 = 49417943; - partContext.PartId = 1; - partContext.PartETag = new PartETag(3, "1234567890"); - - string s = partContext.ToString(); - - ResumablePartContext partContext2 = new ResumablePartContext(); - partContext2.FromString(s); - Assert.AreEqual(partContext.Position, partContext2.Position); - Assert.AreEqual(partContext.Length, partContext2.Length); - Assert.AreEqual(partContext.IsCompleted, partContext2.IsCompleted); - Assert.AreEqual(partContext.Crc64, partContext2.Crc64); - Assert.AreEqual(partContext.PartETag.ETag, partContext2.PartETag.ETag); - Assert.AreEqual(partContext.PartETag.PartNumber, partContext2.PartETag.PartNumber); - } - - [Test] - public void ResumableContextSerializationTest() - { - ResumableContextSerializationTest("testMd5"); - } - - [Test] - public void ResumableContextWithoutMd5SerializationTest() - { - ResumableContextSerializationTest(null); - } - - public void ResumableContextSerializationTest(string md5) - { - ResumableContext resumableContext = new ResumableContext("bk1", "key1", "chkdir"); - resumableContext.UploadId = "UploadId"; - resumableContext.PartContextList = new List(); - ResumablePartContext partContext = new ResumablePartContext(); - partContext.Position = 1000; - partContext.Length = 1024 * 100; - partContext.IsCompleted = true; - partContext.Crc64 = 49417943; - partContext.PartId = 1; - partContext.PartETag = new PartETag(3, "1234567890"); - - resumableContext.PartContextList.Add(partContext); - resumableContext.ContentMd5 = md5; - resumableContext.Crc64 = "1234567890"; - - string s = resumableContext.ToString(); - - ResumableContext resumableContext2 = new ResumableContext("bk2", "key2", "chdir2"); - resumableContext2.FromString(s); - Assert.AreEqual(resumableContext.ContentMd5, resumableContext2.ContentMd5); - Assert.AreEqual(resumableContext.Crc64, resumableContext2.Crc64); - Assert.AreEqual(resumableContext.PartContextList.Count, resumableContext2.PartContextList.Count); - - ResumablePartContext partContext2 = resumableContext2.PartContextList[0]; - Assert.AreEqual(partContext.Position, partContext2.Position); - Assert.AreEqual(partContext.Length, partContext2.Length); - Assert.AreEqual(partContext.IsCompleted, partContext2.IsCompleted); - Assert.AreEqual(partContext.Crc64, partContext2.Crc64); - Assert.AreEqual(partContext.PartETag.ETag, partContext2.PartETag.ETag); - Assert.AreEqual(partContext.PartETag.PartNumber, partContext2.PartETag.PartNumber); - } - } -} diff --git a/test/aliyun-oss-sdk-test.csproj b/test/aliyun-oss-sdk-test.csproj index c8ca1e5..548d904 100644 --- a/test/aliyun-oss-sdk-test.csproj +++ b/test/aliyun-oss-sdk-test.csproj @@ -74,8 +74,6 @@ - -