diff --git a/.changes/nextrelease/feature-progress-tracker.json b/.changes/nextrelease/feature-progress-tracker.json
new file mode 100644
index 0000000000..cd9b2ac890
--- /dev/null
+++ b/.changes/nextrelease/feature-progress-tracker.json
@@ -0,0 +1,7 @@
+[
+ {
+ "type": "enhancement",
+ "category": "S3",
+ "description": "'track_upload' configuration option added for MultipartUpload and MultipartCopy"
+ }
+]
diff --git a/src/Multipart/AbstractUploadManager.php b/src/Multipart/AbstractUploadManager.php
index e663245e0d..1ac452493f 100644
--- a/src/Multipart/AbstractUploadManager.php
+++ b/src/Multipart/AbstractUploadManager.php
@@ -49,6 +49,9 @@ abstract class AbstractUploadManager implements Promise\PromisorInterface
/** @var UploadState State used to manage the upload. */
protected $state;
+ /** @var bool Configuration used to indicate if upload progress will be displayed. */
+ protected $displayProgress;
+
/**
* @param Client $client
* @param array $config
@@ -59,6 +62,12 @@ public function __construct(Client $client, array $config = [])
$this->info = $this->loadUploadWorkflowInfo();
$this->config = $config + self::$defaultConfig;
$this->state = $this->determineState();
+
+ if (isset($config['display_progress'])
+ && is_bool($config['display_progress'])
+ ) {
+ $this->displayProgress = $config['display_progress'];
+ }
}
/**
@@ -238,7 +247,7 @@ private function determineState(): UploadState
}
$id[$param] = $this->config[$key];
}
- $state = new UploadState($id);
+ $state = new UploadState($id, $this->config);
$state->setPartSize($this->determinePartSize());
return $state;
diff --git a/src/Multipart/AbstractUploader.php b/src/Multipart/AbstractUploader.php
index 75e6794660..306be14f11 100644
--- a/src/Multipart/AbstractUploader.php
+++ b/src/Multipart/AbstractUploader.php
@@ -12,6 +12,9 @@ abstract class AbstractUploader extends AbstractUploadManager
/** @var Stream Source of the data to be uploaded. */
protected $source;
+ /** @var bool Configuration used to indicate if upload progress will be displayed. */
+ protected $displayProgress;
+
/**
* @param Client $client
* @param mixed $source
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index 4108c4f13b..c3b4243e43 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -13,6 +13,18 @@ class UploadState
const INITIATED = 1;
const COMPLETED = 2;
+ private $progressBar = [
+ "Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n",
+ "|======= | 37.5%\n",
+ "|========== | 50.0%\n",
+ "|============ | 62.5%\n",
+ "|=============== | 75.0%\n",
+ "|================= | 87.5%\n",
+ "|====================| 100.0%\nTransfer complete!\n"
+ ];
+
/** @var array Params used to identity the upload. */
private $id;
@@ -25,12 +37,24 @@ class UploadState
/** @var int Identifies the status the upload. */
private $status = self::CREATED;
+ /** @var array Thresholds for progress of the upload. */
+ private $progressThresholds = [];
+
+ /** @var boolean Determines status for tracking the upload */
+ private $displayProgress = false;
+
/**
* @param array $id Params used to identity the upload.
*/
- public function __construct(array $id)
+ public function __construct(array $id, array $config = [])
{
$this->id = $id;
+
+ if (isset($config['display_progress'])
+ && is_bool($config['display_progress'])
+ ) {
+ $this->displayProgress = $config['display_progress'];
+ }
}
/**
@@ -45,7 +69,7 @@ public function getId()
}
/**
- * Set's the "upload_id", or 3rd part of the upload's ID. This typically
+ * Sets the "upload_id", or 3rd part of the upload's ID. This typically
* only needs to be done after initiating an upload.
*
* @param string $key The param key of the upload_id.
@@ -76,6 +100,52 @@ public function setPartSize($partSize)
$this->partSize = $partSize;
}
+ /**
+ * Sets the 1/8th thresholds array. $totalSize is only sent if
+ * 'track_upload' is true.
+ *
+ * @param $totalSize numeric Size of object to upload.
+ *
+ * @return array
+ */
+ public function setProgressThresholds($totalSize): array
+ {
+ if(!is_numeric($totalSize)) {
+ throw new \InvalidArgumentException(
+ 'The total size of the upload must be a number.'
+ );
+ }
+
+ $this->progressThresholds[0] = 0;
+ for ($i = 1; $i <= 8; $i++) {
+ $this->progressThresholds[] = round($totalSize * ($i / 8));
+ }
+ return $this->progressThresholds;
+ }
+
+ /**
+ * Prints progress of upload.
+ *
+ * @param $totalUploaded numeric Size of upload so far.
+ */
+ public function getDisplayProgress($totalUploaded): void
+ {
+ if (!is_numeric($totalUploaded)) {
+ throw new \InvalidArgumentException(
+ 'The size of the bytes being uploaded must be a number.'
+ );
+ }
+
+ if ($this->displayProgress) {
+ while (!empty($this->progressBar)
+ && $totalUploaded >= $this->progressThresholds[0]
+ ) {
+ echo array_shift($this->progressBar);
+ array_shift($this->progressThresholds);
+ }
+ }
+ }
+
/**
* Marks a part as being uploaded.
*
@@ -108,7 +178,6 @@ public function hasPartBeenUploaded($partNumber)
public function getUploadedParts()
{
ksort($this->uploadedParts);
-
return $this->uploadedParts;
}
diff --git a/src/S3/MultipartCopy.php b/src/S3/MultipartCopy.php
index 5b26dea79e..6a7bc04dd2 100644
--- a/src/S3/MultipartCopy.php
+++ b/src/S3/MultipartCopy.php
@@ -52,6 +52,8 @@ class MultipartCopy extends AbstractUploadManager
* options are ignored.
* - source_metadata: (Aws\ResultInterface) An object that represents the
* result of executing a HeadObject command on the copy source.
+ * - display_progress: (boolean) Set true to track status in 1/8th increments
+ * for upload.
*
* @param S3ClientInterface $client Client used for the upload.
* @param string|array $source Location of the data to be copied (in the
@@ -75,6 +77,12 @@ public function __construct(
$client,
array_change_key_case($config) + ['source_metadata' => null]
);
+
+ if ($this->displayProgress) {
+ $this->getState()->setProgressThresholds(
+ $this->sourceMetadata["ContentLength"]
+ );
+ }
}
/**
@@ -238,5 +246,4 @@ private function getInputSource($inputSource)
$sourceBuilder .= ltrim(rawurldecode($inputSource), '/');
return $sourceBuilder;
}
-
}
diff --git a/src/S3/MultipartUploader.php b/src/S3/MultipartUploader.php
index ae47d7e5fd..b2bed2b603 100644
--- a/src/S3/MultipartUploader.php
+++ b/src/S3/MultipartUploader.php
@@ -55,6 +55,8 @@ class MultipartUploader extends AbstractUploader
* of the multipart upload and that is used to resume a previous upload.
* When this option is provided, the `bucket`, `key`, and `part_size`
* options are ignored.
+ * - track_upload: (boolean) Set true to track status in 1/8th increments
+ * for upload.
*
* @param S3ClientInterface $client Client used for the upload.
* @param mixed $source Source of the data to upload.
@@ -70,6 +72,10 @@ public function __construct(
'key' => null,
'exception_class' => S3MultipartUploadException::class,
]);
+
+ if ($this->displayProgress) {
+ $this->getState()->setProgressThresholds($this->source->getSize());
+ }
}
protected function loadUploadWorkflowInfo()
diff --git a/src/S3/MultipartUploadingTrait.php b/src/S3/MultipartUploadingTrait.php
index 002bd43c46..541bca8592 100644
--- a/src/S3/MultipartUploadingTrait.php
+++ b/src/S3/MultipartUploadingTrait.php
@@ -7,6 +7,8 @@
trait MultipartUploadingTrait
{
+ private $uploadedBytes = 0;
+
/**
* Creates an UploadState object for a multipart upload by querying the
* service for the specified upload's information.
@@ -59,6 +61,13 @@ protected function handleResult(CommandInterface $command, ResultInterface $resu
$partData[$checksumMemberName] = $result[$checksumMemberName];
}
$this->getState()->markPartAsUploaded($command['PartNumber'], $partData);
+
+ // Updates counter for uploaded bytes.
+ $this->uploadedBytes += $command["ContentLength"];
+ // Sends uploaded bytes to progress tracker if getDisplayProgress set
+ if ($this->displayProgress) {
+ $this->getState()->getDisplayProgress($this->uploadedBytes);
+ }
}
abstract protected function extractETag(ResultInterface $result);
diff --git a/tests/Multipart/UploadStateTest.php b/tests/Multipart/UploadStateTest.php
index 64ac615f30..0d810b0d20 100644
--- a/tests/Multipart/UploadStateTest.php
+++ b/tests/Multipart/UploadStateTest.php
@@ -2,7 +2,7 @@
namespace Aws\Test\Multipart;
use Aws\Multipart\UploadState;
-use PHPUnit\Framework\TestCase;
+use Yoast\PHPUnitPolyfills\TestCases\TestCase;
/**
* @covers Aws\Multipart\UploadState
@@ -70,4 +70,159 @@ public function testSerializationWorks()
$this->assertTrue($newState->isInitiated());
$this->assertArrayHasKey('foo', $newState->getId());
}
+
+ public function testEmptyUploadStateOutputWithConfigFalse()
+ {
+ $state = new UploadState([], ['display_progress' => false]);
+ $state->getDisplayProgress(13);
+ $this->expectOutputString('');
+ }
+
+ /**
+ * @dataProvider getDisplayProgressCases
+ */
+ public function testGetDisplayProgressPrintsProgress(
+ $totalSize,
+ $totalUploaded,
+ $progressBar
+ ) {
+ $state = new UploadState([], ['display_progress' => true]);
+ $state->setProgressThresholds($totalSize);
+ $state->getDisplayProgress($totalUploaded);
+
+ $this->expectOutputString($progressBar);
+ }
+
+ public function getDisplayProgressCases()
+ {
+ $progressBar = ["Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n",
+ "|======= | 37.5%\n",
+ "|========== | 50.0%\n",
+ "|============ | 62.5%\n",
+ "|=============== | 75.0%\n",
+ "|================= | 87.5%\n",
+ "|====================| 100.0%\nTransfer complete!\n"];
+ return [
+ [100000, 0, $progressBar[0]],
+ [100000, 12499, $progressBar[0]],
+ [100000, 12500, "{$progressBar[0]}{$progressBar[1]}"],
+ [100000, 24999, "{$progressBar[0]}{$progressBar[1]}"],
+ [100000, 25000, "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}"],
+ [100000, 37499, "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}"],
+ [
+ 100000,
+ 37500,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}"
+ ],
+ [
+ 100000,
+ 49999,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}"
+ ],
+ [
+ 100000,
+ 50000,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" .
+ "{$progressBar[4]}"
+ ],
+ [
+ 100000,
+ 62499,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" .
+ "{$progressBar[4]}"
+ ],
+ [
+ 100000,
+ 62500,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" .
+ "{$progressBar[4]}{$progressBar[5]}"
+ ],
+ [
+ 100000,
+ 74999,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" .
+ "{$progressBar[4]}{$progressBar[5]}"
+ ],
+ [
+ 100000,
+ 75000,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" .
+ "{$progressBar[4]}{$progressBar[5]}{$progressBar[6]}"
+ ],
+ [
+ 100000,
+ 87499,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" .
+ "{$progressBar[4]}{$progressBar[5]}{$progressBar[6]}"
+ ],
+ [
+ 100000,
+ 87500,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" .
+ "{$progressBar[4]}{$progressBar[5]}{$progressBar[6]}{$progressBar[7]}"
+ ],
+ [
+ 100000,
+ 99999,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}" .
+ "{$progressBar[4]}{$progressBar[5]}{$progressBar[6]}{$progressBar[7]}"
+ ],
+ [100000, 100000, implode($progressBar)]
+ ];
+ }
+
+ /**
+ * @dataProvider getThresholdCases
+ */
+ public function testUploadThresholds($totalSize)
+ {
+ $state = new UploadState([]);
+ $threshold = $state->setProgressThresholds($totalSize);
+
+ $this->assertIsArray($threshold);
+ $this->assertCount(9, $threshold);
+ }
+
+ public function getThresholdCases()
+ {
+ return [
+ [0],
+ [100000],
+ [100001]
+ ];
+ }
+
+ /**
+ * @dataProvider getInvalidIntCases
+ */
+ public function testSetProgressThresholdsThrowsException($totalSize)
+ {
+ $state = new UploadState([]);
+ $this->expectExceptionMessage('The total size of the upload must be a number.');
+ $this->expectException(\InvalidArgumentException::class);
+
+ $state->setProgressThresholds($totalSize);
+ }
+
+ /**
+ * @dataProvider getInvalidIntCases
+ */
+ public function testDisplayProgressThrowsException($totalUploaded)
+ {
+ $state = new UploadState([]);
+ $this->expectExceptionMessage('The size of the bytes being uploaded must be a number.');
+ $this->expectException(\InvalidArgumentException::class);
+ $state->getDisplayProgress($totalUploaded);
+ }
+
+ public function getInvalidIntCases()
+ {
+ return [
+ [''],
+ [null],
+ ['aws']
+ ];
+ }
}
diff --git a/tests/S3/MultipartCopyTest.php b/tests/S3/MultipartCopyTest.php
index f217f594af..74dcbedeeb 100644
--- a/tests/S3/MultipartCopyTest.php
+++ b/tests/S3/MultipartCopyTest.php
@@ -1,10 +1,13 @@
assertTrue($uploader->getState()->isCompleted());
$this->assertSame($url, $result['ObjectURL']);
}
+
+ public function testCopyPrintsProgress()
+ {
+ $progressBar = [
+ "Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n",
+ "|======= | 37.5%\n",
+ "|========== | 50.0%\n",
+ "|============ | 62.5%\n",
+ "|=============== | 75.0%\n",
+ "|================= | 87.5%\n",
+ "|====================| 100.0%\nTransfer complete!\n"
+ ];
+ $client = $this->getTestClient('s3');
+ $copyOptions = [
+ 'bucket' => 'foo',
+ 'key' => 'bar',
+ 'display_progress' => true,
+ 'source_metadata' => new Result(['ContentLength' => 11 * self::MB])
+ ];
+ $url = 'http://foo.s3.amazonaws.com/bar';
+
+ $this->expectOutputString(implode("", $progressBar));
+ $this->addMockResults($client, [
+ new Result(['UploadId' => 'baz']),
+ new Result(['ETag' => 'A']),
+ new Result(['ETag' => 'B']),
+ new Result(['ETag' => 'C']),
+ new Result(['Location' => $url])
+ ]);
+
+ $uploader = new MultipartCopy($client, '/bucket/key', $copyOptions);
+ $result = $uploader->upload();
+
+ $this->assertTrue($uploader->getState()->isCompleted());
+ $this->assertSame($url, $result['ObjectURL']);
+ }
+
+ public function testFailedCopyPrintsPartialProgress()
+ {
+ $partialBar = [
+ "Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n"
+ ];
+ $this->expectOutputString(implode("", $partialBar));
+
+ $this->expectExceptionMessage(
+ "An exception occurred while uploading parts to a multipart upload"
+ );
+ $this->expectException(MultipartUploadException::class);
+ $counter = 0;
+
+ $httpHandler = function ($request, array $options) use (&$counter) {
+ if ($counter < 4) {
+ $body = "" .
+ "baz";
+ } else {
+ $body = "\n\n\n";
+ }
+ $counter++;
+
+ return Promise\Create::promiseFor(
+ new Psr7\Response(200, [], $body)
+ );
+ };
+
+ $client = $this->getTestClient('s3', ['http_handler' => $httpHandler]);
+ $copyOptions = [
+ 'bucket' => 'foo',
+ 'key' => 'bar',
+ 'display_progress' => true,
+ 'source_metadata' => new Result(['ContentLength' => 50 * self::MB])
+ ];
+
+ $uploader = new MultipartCopy($client, '/bucket/key', $copyOptions);
+ $uploader->upload();
+ }
}
diff --git a/tests/S3/MultipartUploaderTest.php b/tests/S3/MultipartUploaderTest.php
index 6abfaa8277..a21522e0c5 100644
--- a/tests/S3/MultipartUploaderTest.php
+++ b/tests/S3/MultipartUploaderTest.php
@@ -317,6 +317,7 @@ public function testAppliesAmbiguousSuccessParsing()
$uploader->upload();
}
+
/**
* @dataProvider testMultipartSuccessStreams
*/
@@ -363,4 +364,97 @@ public function testUploaderAddsFlexibleChecksums($stream, $size)
$this->assertSame('xyz', $result['ChecksumSHA256']);
$this->assertSame($url, $result['ObjectURL']);
}
+
+ public function testUploadPrintsProgress()
+ {
+ $progressBar = [
+ "Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n",
+ "|======= | 37.5%\n",
+ "|========== | 50.0%\n",
+ "|============ | 62.5%\n",
+ "|=============== | 75.0%\n",
+ "|================= | 87.5%\n",
+ "|====================| 100.0%\nTransfer complete!\n"
+ ];
+ $client = $this->getTestClient('s3');
+ $uploadOptions = [
+ 'bucket' => 'foo',
+ 'key' => 'bar',
+ 'display_progress' => true
+ ];
+ $this->expectOutputString(implode("", $progressBar));
+ $url = 'http://foo.s3.amazonaws.com/bar';
+
+ $size = 12 * self::MB;
+ $data = str_repeat('.', $size);
+ $filename = sys_get_temp_dir() . '/' . self::FILENAME;
+ file_put_contents($filename, $data);
+ $stream = Psr7\Utils::streamFor(fopen($filename, 'r'));
+
+ $this->addMockResults($client, [
+ new Result(['UploadId' => 'baz']),
+ new Result(['ETag' => 'A']),
+ new Result(['ETag' => 'B']),
+ new Result(['ETag' => 'C']),
+ new Result(['Location' => $url])
+ ]);
+
+ $uploader = new MultipartUploader($client, $stream, $uploadOptions);
+ $result = $uploader->upload();
+
+ $this->assertTrue($uploader->getState()->isCompleted());
+ $this->assertSame($url, $result['ObjectURL']);
+ }
+
+ public function testFailedUploadPrintsPartialProgress()
+ {
+ $partialBar = [
+ "Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n"
+ ];
+ $this->expectOutputString(implode("", $partialBar));
+
+ $this->expectExceptionMessage(
+ "An exception occurred while uploading parts to a multipart upload"
+ );
+ $this->expectException(\Aws\S3\Exception\S3MultipartUploadException::class);
+ $counter = 0;
+
+ $httpHandler = function ($request, array $options) use (&$counter) {
+ if ($counter < 4) {
+ $body = "" .
+ "baz";
+ } else {
+ $body = "\n\n\n";
+ }
+ $counter++;
+
+ return Promise\Create::promiseFor(
+ new Psr7\Response(200, [], $body)
+ );
+ };
+
+ $s3 = new S3Client([
+ 'version' => 'latest',
+ 'region' => 'us-east-1',
+ 'http_handler' => $httpHandler
+ ]);
+
+ $data = str_repeat('.', 50 * self::MB);
+ $source = Psr7\Utils::streamFor($data);
+
+ $uploader = new MultipartUploader(
+ $s3,
+ $source,
+ [
+ 'bucket' => 'test-bucket',
+ 'key' => 'test-key',
+ 'display_progress' => true
+ ]
+ );
+ $uploader->upload();
+ }
}