diff --git a/.changes/nextrelease/feature-progress-tracker.json b/.changes/nextrelease/feature-progress-tracker.json new file mode 100644 index 0000000000..cd9b2ac890 --- /dev/null +++ b/.changes/nextrelease/feature-progress-tracker.json @@ -0,0 +1,7 @@ +[ + { + "type": "enhancement", + "category": "S3", + "description": "'track_upload' configuration option added for MultipartUpload and MultipartCopy" + } +] diff --git a/src/Multipart/AbstractUploadManager.php b/src/Multipart/AbstractUploadManager.php index e663245e0d..1ac452493f 100644 --- a/src/Multipart/AbstractUploadManager.php +++ b/src/Multipart/AbstractUploadManager.php @@ -49,6 +49,9 @@ abstract class AbstractUploadManager implements Promise\PromisorInterface /** @var UploadState State used to manage the upload. */ protected $state; + /** @var bool Configuration used to indicate if upload progress will be displayed. */ + protected $displayProgress; + /** * @param Client $client * @param array $config @@ -59,6 +62,12 @@ public function __construct(Client $client, array $config = []) $this->info = $this->loadUploadWorkflowInfo(); $this->config = $config + self::$defaultConfig; $this->state = $this->determineState(); + + if (isset($config['display_progress']) + && is_bool($config['display_progress']) + ) { + $this->displayProgress = $config['display_progress']; + } } /** @@ -238,7 +247,7 @@ private function determineState(): UploadState } $id[$param] = $this->config[$key]; } - $state = new UploadState($id); + $state = new UploadState($id, $this->config); $state->setPartSize($this->determinePartSize()); return $state; diff --git a/src/Multipart/AbstractUploader.php b/src/Multipart/AbstractUploader.php index 75e6794660..306be14f11 100644 --- a/src/Multipart/AbstractUploader.php +++ b/src/Multipart/AbstractUploader.php @@ -12,6 +12,9 @@ abstract class AbstractUploader extends AbstractUploadManager /** @var Stream Source of the data to be uploaded. */ protected $source; + /** @var bool Configuration used to indicate if upload progress will be displayed. */ + protected $displayProgress; + /** * @param Client $client * @param mixed $source diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php index 4108c4f13b..c3b4243e43 100644 --- a/src/Multipart/UploadState.php +++ b/src/Multipart/UploadState.php @@ -13,6 +13,18 @@ class UploadState const INITIATED = 1; const COMPLETED = 2; + private $progressBar = [ + "Transfer initiated...\n| | 0.0%\n", + "|== | 12.5%\n", + "|===== | 25.0%\n", + "|======= | 37.5%\n", + "|========== | 50.0%\n", + "|============ | 62.5%\n", + "|=============== | 75.0%\n", + "|================= | 87.5%\n", + "|====================| 100.0%\nTransfer complete!\n" + ]; + /** @var array Params used to identity the upload. */ private $id; @@ -25,12 +37,24 @@ class UploadState /** @var int Identifies the status the upload. */ private $status = self::CREATED; + /** @var array Thresholds for progress of the upload. */ + private $progressThresholds = []; + + /** @var boolean Determines status for tracking the upload */ + private $displayProgress = false; + /** * @param array $id Params used to identity the upload. */ - public function __construct(array $id) + public function __construct(array $id, array $config = []) { $this->id = $id; + + if (isset($config['display_progress']) + && is_bool($config['display_progress']) + ) { + $this->displayProgress = $config['display_progress']; + } } /** @@ -45,7 +69,7 @@ public function getId() } /** - * Set's the "upload_id", or 3rd part of the upload's ID. This typically + * Sets the "upload_id", or 3rd part of the upload's ID. This typically * only needs to be done after initiating an upload. * * @param string $key The param key of the upload_id. @@ -76,6 +100,52 @@ public function setPartSize($partSize) $this->partSize = $partSize; } + /** + * Sets the 1/8th thresholds array. $totalSize is only sent if + * 'track_upload' is true. + * + * @param $totalSize numeric Size of object to upload. + * + * @return array + */ + public function setProgressThresholds($totalSize): array + { + if(!is_numeric($totalSize)) { + throw new \InvalidArgumentException( + 'The total size of the upload must be a number.' + ); + } + + $this->progressThresholds[0] = 0; + for ($i = 1; $i <= 8; $i++) { + $this->progressThresholds[] = round($totalSize * ($i / 8)); + } + return $this->progressThresholds; + } + + /** + * Prints progress of upload. + * + * @param $totalUploaded numeric Size of upload so far. + */ + public function getDisplayProgress($totalUploaded): void + { + if (!is_numeric($totalUploaded)) { + throw new \InvalidArgumentException( + 'The size of the bytes being uploaded must be a number.' + ); + } + + if ($this->displayProgress) { + while (!empty($this->progressBar) + && $totalUploaded >= $this->progressThresholds[0] + ) { + echo array_shift($this->progressBar); + array_shift($this->progressThresholds); + } + } + } + /** * Marks a part as being uploaded. * @@ -108,7 +178,6 @@ public function hasPartBeenUploaded($partNumber) public function getUploadedParts() { ksort($this->uploadedParts); - return $this->uploadedParts; } diff --git a/src/S3/MultipartCopy.php b/src/S3/MultipartCopy.php index 5b26dea79e..6a7bc04dd2 100644 --- a/src/S3/MultipartCopy.php +++ b/src/S3/MultipartCopy.php @@ -52,6 +52,8 @@ class MultipartCopy extends AbstractUploadManager * options are ignored. * - source_metadata: (Aws\ResultInterface) An object that represents the * result of executing a HeadObject command on the copy source. + * - display_progress: (boolean) Set true to track status in 1/8th increments + * for upload. * * @param S3ClientInterface $client Client used for the upload. * @param string|array $source Location of the data to be copied (in the @@ -75,6 +77,12 @@ public function __construct( $client, array_change_key_case($config) + ['source_metadata' => null] ); + + if ($this->displayProgress) { + $this->getState()->setProgressThresholds( + $this->sourceMetadata["ContentLength"] + ); + } } /** @@ -238,5 +246,4 @@ private function getInputSource($inputSource) $sourceBuilder .= ltrim(rawurldecode($inputSource), '/'); return $sourceBuilder; } - } diff --git a/src/S3/MultipartUploader.php b/src/S3/MultipartUploader.php index ae47d7e5fd..b2bed2b603 100644 --- a/src/S3/MultipartUploader.php +++ b/src/S3/MultipartUploader.php @@ -55,6 +55,8 @@ class MultipartUploader extends AbstractUploader * of the multipart upload and that is used to resume a previous upload. * When this option is provided, the `bucket`, `key`, and `part_size` * options are ignored. + * - track_upload: (boolean) Set true to track status in 1/8th increments + * for upload. * * @param S3ClientInterface $client Client used for the upload. * @param mixed $source Source of the data to upload. @@ -70,6 +72,10 @@ public function __construct( 'key' => null, 'exception_class' => S3MultipartUploadException::class, ]); + + if ($this->displayProgress) { + $this->getState()->setProgressThresholds($this->source->getSize()); + } } protected function loadUploadWorkflowInfo() diff --git a/src/S3/MultipartUploadingTrait.php b/src/S3/MultipartUploadingTrait.php index 002bd43c46..541bca8592 100644 --- a/src/S3/MultipartUploadingTrait.php +++ b/src/S3/MultipartUploadingTrait.php @@ -7,6 +7,8 @@ trait MultipartUploadingTrait { + private $uploadedBytes = 0; + /** * Creates an UploadState object for a multipart upload by querying the * service for the specified upload's information. @@ -59,6 +61,13 @@ protected function handleResult(CommandInterface $command, ResultInterface $resu $partData[$checksumMemberName] = $result[$checksumMemberName]; } $this->getState()->markPartAsUploaded($command['PartNumber'], $partData); + + // Updates counter for uploaded bytes. + $this->uploadedBytes += $command["ContentLength"]; + // Sends uploaded bytes to progress tracker if getDisplayProgress set + if ($this->displayProgress) { + $this->getState()->getDisplayProgress($this->uploadedBytes); + } } abstract protected function extractETag(ResultInterface $result); diff --git a/tests/Multipart/UploadStateTest.php b/tests/Multipart/UploadStateTest.php index 64ac615f30..0d810b0d20 100644 --- a/tests/Multipart/UploadStateTest.php +++ b/tests/Multipart/UploadStateTest.php @@ -2,7 +2,7 @@ namespace Aws\Test\Multipart; use Aws\Multipart\UploadState; -use PHPUnit\Framework\TestCase; +use Yoast\PHPUnitPolyfills\TestCases\TestCase; /** * @covers Aws\Multipart\UploadState @@ -70,4 +70,159 @@ public function testSerializationWorks() $this->assertTrue($newState->isInitiated()); $this->assertArrayHasKey('foo', $newState->getId()); } + + public function testEmptyUploadStateOutputWithConfigFalse() + { + $state = new UploadState([], ['display_progress' => false]); + $state->getDisplayProgress(13); + $this->expectOutputString(''); + } + + /** + * @dataProvider getDisplayProgressCases + */ + public function testGetDisplayProgressPrintsProgress( + $totalSize, + $totalUploaded, + $progressBar + ) { + $state = new UploadState([], ['display_progress' => true]); + $state->setProgressThresholds($totalSize); + $state->getDisplayProgress($totalUploaded); + + $this->expectOutputString($progressBar); + } + + public function getDisplayProgressCases() + { + $progressBar = ["Transfer initiated...\n| | 0.0%\n", + "|== | 12.5%\n", + "|===== | 25.0%\n", + "|======= | 37.5%\n", + "|========== | 50.0%\n", + "|============ | 62.5%\n", + "|=============== | 75.0%\n", + "|================= | 87.5%\n", + "|====================| 100.0%\nTransfer complete!\n"]; + return [ + [100000, 0, $progressBar[0]], + [100000, 12499, $progressBar[0]], + [100000, 12500, "{$progressBar[0]}{$progressBar[1]}"], + [100000, 24999, "{$progressBar[0]}{$progressBar[1]}"], + [100000, 25000, "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}"], + [100000, 37499, "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}"], + [ + 100000, + 37500, + "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" + ], + [ + 100000, + 49999, + "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" + ], + [ + 100000, + 50000, + "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" . + "{$progressBar[4]}" + ], + [ + 100000, + 62499, + "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" . + "{$progressBar[4]}" + ], + [ + 100000, + 62500, + "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" . + "{$progressBar[4]}{$progressBar[5]}" + ], + [ + 100000, + 74999, + "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" . + "{$progressBar[4]}{$progressBar[5]}" + ], + [ + 100000, + 75000, + "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" . + "{$progressBar[4]}{$progressBar[5]}{$progressBar[6]}" + ], + [ + 100000, + 87499, + "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" . + "{$progressBar[4]}{$progressBar[5]}{$progressBar[6]}" + ], + [ + 100000, + 87500, + "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" . + "{$progressBar[4]}{$progressBar[5]}{$progressBar[6]}{$progressBar[7]}" + ], + [ + 100000, + 99999, + "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" . + "{$progressBar[4]}{$progressBar[5]}{$progressBar[6]}{$progressBar[7]}" + ], + [100000, 100000, implode($progressBar)] + ]; + } + + /** + * @dataProvider getThresholdCases + */ + public function testUploadThresholds($totalSize) + { + $state = new UploadState([]); + $threshold = $state->setProgressThresholds($totalSize); + + $this->assertIsArray($threshold); + $this->assertCount(9, $threshold); + } + + public function getThresholdCases() + { + return [ + [0], + [100000], + [100001] + ]; + } + + /** + * @dataProvider getInvalidIntCases + */ + public function testSetProgressThresholdsThrowsException($totalSize) + { + $state = new UploadState([]); + $this->expectExceptionMessage('The total size of the upload must be a number.'); + $this->expectException(\InvalidArgumentException::class); + + $state->setProgressThresholds($totalSize); + } + + /** + * @dataProvider getInvalidIntCases + */ + public function testDisplayProgressThrowsException($totalUploaded) + { + $state = new UploadState([]); + $this->expectExceptionMessage('The size of the bytes being uploaded must be a number.'); + $this->expectException(\InvalidArgumentException::class); + $state->getDisplayProgress($totalUploaded); + } + + public function getInvalidIntCases() + { + return [ + [''], + [null], + ['aws'] + ]; + } } diff --git a/tests/S3/MultipartCopyTest.php b/tests/S3/MultipartCopyTest.php index f217f594af..74dcbedeeb 100644 --- a/tests/S3/MultipartCopyTest.php +++ b/tests/S3/MultipartCopyTest.php @@ -1,10 +1,13 @@ assertTrue($uploader->getState()->isCompleted()); $this->assertSame($url, $result['ObjectURL']); } + + public function testCopyPrintsProgress() + { + $progressBar = [ + "Transfer initiated...\n| | 0.0%\n", + "|== | 12.5%\n", + "|===== | 25.0%\n", + "|======= | 37.5%\n", + "|========== | 50.0%\n", + "|============ | 62.5%\n", + "|=============== | 75.0%\n", + "|================= | 87.5%\n", + "|====================| 100.0%\nTransfer complete!\n" + ]; + $client = $this->getTestClient('s3'); + $copyOptions = [ + 'bucket' => 'foo', + 'key' => 'bar', + 'display_progress' => true, + 'source_metadata' => new Result(['ContentLength' => 11 * self::MB]) + ]; + $url = 'http://foo.s3.amazonaws.com/bar'; + + $this->expectOutputString(implode("", $progressBar)); + $this->addMockResults($client, [ + new Result(['UploadId' => 'baz']), + new Result(['ETag' => 'A']), + new Result(['ETag' => 'B']), + new Result(['ETag' => 'C']), + new Result(['Location' => $url]) + ]); + + $uploader = new MultipartCopy($client, '/bucket/key', $copyOptions); + $result = $uploader->upload(); + + $this->assertTrue($uploader->getState()->isCompleted()); + $this->assertSame($url, $result['ObjectURL']); + } + + public function testFailedCopyPrintsPartialProgress() + { + $partialBar = [ + "Transfer initiated...\n| | 0.0%\n", + "|== | 12.5%\n", + "|===== | 25.0%\n" + ]; + $this->expectOutputString(implode("", $partialBar)); + + $this->expectExceptionMessage( + "An exception occurred while uploading parts to a multipart upload" + ); + $this->expectException(MultipartUploadException::class); + $counter = 0; + + $httpHandler = function ($request, array $options) use (&$counter) { + if ($counter < 4) { + $body = "" . + "baz"; + } else { + $body = "\n\n\n"; + } + $counter++; + + return Promise\Create::promiseFor( + new Psr7\Response(200, [], $body) + ); + }; + + $client = $this->getTestClient('s3', ['http_handler' => $httpHandler]); + $copyOptions = [ + 'bucket' => 'foo', + 'key' => 'bar', + 'display_progress' => true, + 'source_metadata' => new Result(['ContentLength' => 50 * self::MB]) + ]; + + $uploader = new MultipartCopy($client, '/bucket/key', $copyOptions); + $uploader->upload(); + } } diff --git a/tests/S3/MultipartUploaderTest.php b/tests/S3/MultipartUploaderTest.php index 6abfaa8277..a21522e0c5 100644 --- a/tests/S3/MultipartUploaderTest.php +++ b/tests/S3/MultipartUploaderTest.php @@ -317,6 +317,7 @@ public function testAppliesAmbiguousSuccessParsing() $uploader->upload(); } + /** * @dataProvider testMultipartSuccessStreams */ @@ -363,4 +364,97 @@ public function testUploaderAddsFlexibleChecksums($stream, $size) $this->assertSame('xyz', $result['ChecksumSHA256']); $this->assertSame($url, $result['ObjectURL']); } + + public function testUploadPrintsProgress() + { + $progressBar = [ + "Transfer initiated...\n| | 0.0%\n", + "|== | 12.5%\n", + "|===== | 25.0%\n", + "|======= | 37.5%\n", + "|========== | 50.0%\n", + "|============ | 62.5%\n", + "|=============== | 75.0%\n", + "|================= | 87.5%\n", + "|====================| 100.0%\nTransfer complete!\n" + ]; + $client = $this->getTestClient('s3'); + $uploadOptions = [ + 'bucket' => 'foo', + 'key' => 'bar', + 'display_progress' => true + ]; + $this->expectOutputString(implode("", $progressBar)); + $url = 'http://foo.s3.amazonaws.com/bar'; + + $size = 12 * self::MB; + $data = str_repeat('.', $size); + $filename = sys_get_temp_dir() . '/' . self::FILENAME; + file_put_contents($filename, $data); + $stream = Psr7\Utils::streamFor(fopen($filename, 'r')); + + $this->addMockResults($client, [ + new Result(['UploadId' => 'baz']), + new Result(['ETag' => 'A']), + new Result(['ETag' => 'B']), + new Result(['ETag' => 'C']), + new Result(['Location' => $url]) + ]); + + $uploader = new MultipartUploader($client, $stream, $uploadOptions); + $result = $uploader->upload(); + + $this->assertTrue($uploader->getState()->isCompleted()); + $this->assertSame($url, $result['ObjectURL']); + } + + public function testFailedUploadPrintsPartialProgress() + { + $partialBar = [ + "Transfer initiated...\n| | 0.0%\n", + "|== | 12.5%\n", + "|===== | 25.0%\n" + ]; + $this->expectOutputString(implode("", $partialBar)); + + $this->expectExceptionMessage( + "An exception occurred while uploading parts to a multipart upload" + ); + $this->expectException(\Aws\S3\Exception\S3MultipartUploadException::class); + $counter = 0; + + $httpHandler = function ($request, array $options) use (&$counter) { + if ($counter < 4) { + $body = "" . + "baz"; + } else { + $body = "\n\n\n"; + } + $counter++; + + return Promise\Create::promiseFor( + new Psr7\Response(200, [], $body) + ); + }; + + $s3 = new S3Client([ + 'version' => 'latest', + 'region' => 'us-east-1', + 'http_handler' => $httpHandler + ]); + + $data = str_repeat('.', 50 * self::MB); + $source = Psr7\Utils::streamFor($data); + + $uploader = new MultipartUploader( + $s3, + $source, + [ + 'bucket' => 'test-bucket', + 'key' => 'test-key', + 'display_progress' => true + ] + ); + $uploader->upload(); + } }