diff --git a/packages/php-db-import-export/src/Backend/Bigquery/Export/GcsExportAdapter.php b/packages/php-db-import-export/src/Backend/Bigquery/Export/GcsExportAdapter.php index 7accf97e6..f9c77cde3 100644 --- a/packages/php-db-import-export/src/Backend/Bigquery/Export/GcsExportAdapter.php +++ b/packages/php-db-import-export/src/Backend/Bigquery/Export/GcsExportAdapter.php @@ -68,6 +68,7 @@ public function runCopyCommand( if ($exportOptions->generateManifest()) { (new Storage\GCS\ManifestGenerator\GcsSlicedManifestFromFolderGenerator( $destination->getClient(), + new Storage\GCS\ManifestGenerator\WriteStreamFactory(), )) ->generateAndSaveManifest($destination->getRelativePath()); } diff --git a/packages/php-db-import-export/src/Backend/Snowflake/Export/GcsExportAdapter.php b/packages/php-db-import-export/src/Backend/Snowflake/Export/GcsExportAdapter.php index 05e7b8395..44a580808 100644 --- a/packages/php-db-import-export/src/Backend/Snowflake/Export/GcsExportAdapter.php +++ b/packages/php-db-import-export/src/Backend/Snowflake/Export/GcsExportAdapter.php @@ -9,6 +9,7 @@ use Keboola\Db\ImportExport\ExportOptions; use Keboola\Db\ImportExport\ExportOptionsInterface; use Keboola\Db\ImportExport\Storage; +use Keboola\Db\ImportExport\Storage\GCS\ManifestGenerator\WriteStreamFactory; class GcsExportAdapter implements BackendExportAdapterInterface { @@ -73,6 +74,7 @@ public function runCopyCommand( if ($exportOptions->generateManifest()) { (new Storage\GCS\ManifestGenerator\GcsSlicedManifestFromUnloadQueryResultGenerator( $destination->getClient(), + new WriteStreamFactory(), )) ->generateAndSaveManifest($destination->getRelativePath(), $unloadedFiles); } diff --git a/packages/php-db-import-export/src/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromFolderGenerator.php b/packages/php-db-import-export/src/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromFolderGenerator.php index 8851d724e..b28c77a5e 100644 --- a/packages/php-db-import-export/src/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromFolderGenerator.php +++ b/packages/php-db-import-export/src/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromFolderGenerator.php @@ -6,7 +6,6 @@ use Google\Cloud\Storage\StorageClient; use Google\Cloud\Storage\StorageObject; -use Google\Cloud\Storage\WriteStream; use JsonException; use Keboola\Db\ImportExport\Storage\SlicedManifestGeneratorInterface; use Keboola\FileStorage\Path\RelativePathInterface; @@ -16,12 +15,10 @@ */ class GcsSlicedManifestFromFolderGenerator implements SlicedManifestGeneratorInterface { - private StorageClient $client; - public function __construct( - StorageClient $client, + private readonly StorageClient $client, + private readonly WriteStreamFactory $writeStreamFactory, ) { - $this->client = $client; } /** @@ -46,9 +43,7 @@ public function generateAndSaveManifest(RelativePathInterface $path): void 'entries' => $entries, ], JSON_THROW_ON_ERROR); - $writeStream = new WriteStream(null, [ - 'chunkSize' => self::CHUNK_SIZE_256_KB, - ]); + $writeStream = $this->writeStreamFactory->createWriteStream(); $uploader = $bucket->getStreamableUploader($writeStream, [ 'name' => $path->getPathnameWithoutRoot() . 'manifest', ]); diff --git a/packages/php-db-import-export/src/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromUnloadQueryResultGenerator.php b/packages/php-db-import-export/src/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromUnloadQueryResultGenerator.php index f5177d0f0..52bf1083b 100644 --- a/packages/php-db-import-export/src/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromUnloadQueryResultGenerator.php +++ b/packages/php-db-import-export/src/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromUnloadQueryResultGenerator.php @@ -5,7 +5,6 @@ namespace Keboola\Db\ImportExport\Storage\GCS\ManifestGenerator; use Google\Cloud\Storage\StorageClient; -use Google\Cloud\Storage\WriteStream; use JsonException; use Keboola\Db\ImportExport\Storage\SlicedManifestGeneratorInterface; use Keboola\FileStorage\Path\RelativePathInterface; @@ -15,11 +14,10 @@ */ class GcsSlicedManifestFromUnloadQueryResultGenerator implements SlicedManifestGeneratorInterface { - private StorageClient $gcsClient; - - public function __construct(StorageClient $gcsClient) - { - $this->gcsClient = $gcsClient; + public function __construct( + private readonly StorageClient $gcsClient, + private readonly WriteStreamFactory $writeStreamFactory, + ) { } /** @@ -39,9 +37,7 @@ public function generateAndSaveManifest(RelativePathInterface $path, array $list $manifest = [ 'entries' => $entries, ]; - $writeStream = new WriteStream(null, [ - 'chunkSize' => self::CHUNK_SIZE_256_KB, - ]); + $writeStream = $this->writeStreamFactory->createWriteStream(); $uploader = $bucket->getStreamableUploader($writeStream, [ 'name' => $path->getPathnameWithoutRoot() . 'manifest', ]); diff --git a/packages/php-db-import-export/src/Storage/GCS/ManifestGenerator/WriteStreamFactory.php b/packages/php-db-import-export/src/Storage/GCS/ManifestGenerator/WriteStreamFactory.php new file mode 100644 index 000000000..cff51d448 --- /dev/null +++ b/packages/php-db-import-export/src/Storage/GCS/ManifestGenerator/WriteStreamFactory.php @@ -0,0 +1,18 @@ + SlicedManifestGeneratorInterface::CHUNK_SIZE_256_KB, + ]); + } +} diff --git a/packages/php-db-import-export/src/Storage/GCS/SnowflakeExportAdapter.php b/packages/php-db-import-export/src/Storage/GCS/SnowflakeExportAdapter.php index dc42351c6..c7cbb24ba 100644 --- a/packages/php-db-import-export/src/Storage/GCS/SnowflakeExportAdapter.php +++ b/packages/php-db-import-export/src/Storage/GCS/SnowflakeExportAdapter.php @@ -69,6 +69,7 @@ public function runCopyCommand( if ($exportOptions->generateManifest()) { (new Storage\GCS\ManifestGenerator\GcsSlicedManifestFromUnloadQueryResultGenerator( $destination->getClient(), + new Storage\GCS\ManifestGenerator\WriteStreamFactory(), )) ->generateAndSaveManifest($destination->getRelativePath(), $unloadedFiles); } diff --git a/packages/php-db-import-export/tests/unit/Storage/GCS/ManifestGenerator/GCSSlicedManifestFromFolderGeneratorTest.php b/packages/php-db-import-export/tests/unit/Storage/GCS/ManifestGenerator/GCSSlicedManifestFromFolderGeneratorTest.php index f988d3f88..81dd48178 100644 --- a/packages/php-db-import-export/tests/unit/Storage/GCS/ManifestGenerator/GCSSlicedManifestFromFolderGeneratorTest.php +++ b/packages/php-db-import-export/tests/unit/Storage/GCS/ManifestGenerator/GCSSlicedManifestFromFolderGeneratorTest.php @@ -10,7 +10,9 @@ use Google\Cloud\Storage\ObjectIterator; use Google\Cloud\Storage\StorageClient; use Google\Cloud\Storage\StorageObject; +use Google\Cloud\Storage\WriteStream; use Keboola\Db\ImportExport\Storage\GCS\ManifestGenerator\GcsSlicedManifestFromFolderGenerator; +use Keboola\Db\ImportExport\Storage\GCS\ManifestGenerator\WriteStreamFactory; use Keboola\Db\ImportExport\Storage\SlicedManifestGeneratorInterface; use Keboola\FileStorage\Gcs\GcsProvider; use Keboola\FileStorage\Path\RelativePath; @@ -21,6 +23,9 @@ class GCSSlicedManifestFromFolderGeneratorTest extends TestCase { public function testGenerateAndSaveManifest(): void { + $writeStreamMock = $this->createMock(WriteStream::class); + $writeStreamFactory = $this->createMock(WriteStreamFactory::class); + $writeStreamFactory->method('createWriteStream')->willReturn($writeStreamMock); $streamableUploaderMock = $this->getMockBuilder(StreamableUploader::class) ->disableOriginalConstructor() ->getMock(); @@ -47,10 +52,12 @@ public function testGenerateAndSaveManifest(): void $bucket->expects($this->exactly(2)) ->method('name') ->willReturn('bucket1'); - - $bucket->expects($this->once())->method('getStreamableUploader') - ->willReturn($streamableUploaderMock); - + $bucket->method('getStreamableUploader')->willReturnCallback( + function (WriteStream $w, array $o) use ($streamableUploaderMock) { + self::assertSame(['name' => 'prefix/xxxmanifest'], $o); + return $streamableUploaderMock; + }, + ); $clientMock = $this->createMock(StorageClient::class); $clientMock ->expects($this->once()) @@ -60,9 +67,21 @@ public function testGenerateAndSaveManifest(): void $path = RelativePath::createFromRootAndPath(new GcsProvider(), 'bucket1', 'prefix/xxx'); - $generator = new GcsSlicedManifestFromFolderGenerator($clientMock); + $writtenData = ''; + $writeStreamMock->method('write')->willReturnCallback(function (string $data) use (&$writtenData) { + $writtenData .= $data; + return 0; + }); + + $generator = new GcsSlicedManifestFromFolderGenerator($clientMock, $writeStreamFactory); $generator->generateAndSaveManifest($path); + $this->assertSame( + //phpcs:ignore + '{"entries":[{"url":"gs:\\/\\/bucket1\\/prefix\\/xxx\\/obj1_000000.csv","mandatory":true},{"url":"gs:\\/\\/bucket1\\/prefix\\/xxx\\/obj2_000000.csv","mandatory":true}]}', + $writtenData, + ); + $this->assertInstanceOf(SlicedManifestGeneratorInterface::class, $generator); } diff --git a/packages/php-db-import-export/tests/unit/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromUnloadQueryResultGeneratorTest.php b/packages/php-db-import-export/tests/unit/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromUnloadQueryResultGeneratorTest.php index fd17652ef..cd0be3545 100644 --- a/packages/php-db-import-export/tests/unit/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromUnloadQueryResultGeneratorTest.php +++ b/packages/php-db-import-export/tests/unit/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromUnloadQueryResultGeneratorTest.php @@ -7,49 +7,61 @@ use Google\Cloud\Core\Upload\StreamableUploader; use Google\Cloud\Storage\Bucket; use Google\Cloud\Storage\StorageClient; +use Google\Cloud\Storage\WriteStream; use Keboola\Db\ImportExport\Storage\GCS\ManifestGenerator\GcsSlicedManifestFromUnloadQueryResultGenerator; +use Keboola\Db\ImportExport\Storage\GCS\ManifestGenerator\WriteStreamFactory; use Keboola\Db\ImportExport\Storage\SlicedManifestGeneratorInterface; use Keboola\FileStorage\Gcs\GcsProvider; use Keboola\FileStorage\Path\RelativePath; -use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; class GcsSlicedManifestFromUnloadQueryResultGeneratorTest extends TestCase { public function testGenerateAndSaveManifest(): void { - $path = RelativePath::createFromRootAndPath( - new GcsProvider(), - 'tomasfejfar-kbc-services-filestorag-s3filesbucket-ggrrgg35547q', - 'permanent/256/snapshots/in/c-API-tests-e46793dac57ccf8cefb82ae9b8c05844cfabf985/languages/17982.csv.gz', - ); - + $writeStreamMock = $this->createMock(WriteStream::class); + $writeStreamFactory = $this->createMock(WriteStreamFactory::class); + $writeStreamFactory->method('createWriteStream')->willReturn($writeStreamMock); $streamableUploaderMock = $this->getMockBuilder(StreamableUploader::class) ->disableOriginalConstructor() ->getMock(); - $streamableUploaderMock->method('getResumeUri')->willReturn(''); - $streamableUploaderMock->method('upload')->willReturn([]); - $bucketMock = $this->getMockBuilder(Bucket::class) - ->disableOriginalConstructor() - ->setMethods(['upload', 'getStreamableUploader']) - ->getMock(); - $bucketMock->expects($this->once())->method('getStreamableUploader') - ->willReturn($streamableUploaderMock); + $bucketMock = $this->createMock(Bucket::class); + $bucketMock->method('getStreamableUploader')->willReturnCallback( + function (WriteStream $w, array $o) use ($streamableUploaderMock) { + //phpcs:ignore + self::assertSame(['name' => 'permanent/256/snapshots/in/c-API-tests-e46793dac57ccf8cefb82ae9b8c05844cfabf985/languages/17982.csv.gzmanifest'], $o); + return $streamableUploaderMock; + }, + ); - /** @var MockObject|StorageClient $gcsClientMock */ - $gcsClientMock = $this->getMockBuilder(StorageClient::class) - ->disableOriginalConstructor() - ->setMethods(['bucket']) - ->getMock(); + $gcsClientMock = $this->createMock(StorageClient::class); $gcsClientMock->expects($this->once())->method('bucket')->willReturn($bucketMock); - $generator = new GcsSlicedManifestFromUnloadQueryResultGenerator($gcsClientMock); + $path = RelativePath::createFromRootAndPath( + new GcsProvider(), + 'tomasfejfar-kbc-services-filestorag-s3filesbucket-ggrrgg35547q', + 'permanent/256/snapshots/in/c-API-tests-e46793dac57ccf8cefb82ae9b8c05844cfabf985/languages/17982.csv.gz', + ); + + $writtenData = ''; + $writeStreamMock->method('write')->willReturnCallback(function (string $data) use (&$writtenData) { + $writtenData .= $data; + return 0; + }); + + $generator = new GcsSlicedManifestFromUnloadQueryResultGenerator($gcsClientMock, $writeStreamFactory); $generator->generateAndSaveManifest($path, [ ['FILE_NAME' => '17982.csv.gz_0_0_0.csv.gz', 'FILE_SIZE' => '10', 'ROW_COUNT' => '5'], ['FILE_NAME' => '17982.csv.gz_0_0_1.csv.gz', 'FILE_SIZE' => '25', 'ROW_COUNT' => '15'], ]); + $this->assertSame( + //phpcs:ignore + '{"entries":[{"url":"gs:\/\/tomasfejfar-kbc-services-filestorag-s3filesbucket-ggrrgg35547q\/permanent\/256\/snapshots\/in\/c-API-tests-e46793dac57ccf8cefb82ae9b8c05844cfabf985\/languages\/17982.csv.gz_0_0_0.csv.gz","mandatory":true},{"url":"gs:\/\/tomasfejfar-kbc-services-filestorag-s3filesbucket-ggrrgg35547q\/permanent\/256\/snapshots\/in\/c-API-tests-e46793dac57ccf8cefb82ae9b8c05844cfabf985\/languages\/17982.csv.gz_0_0_1.csv.gz","mandatory":true}]}', + $writtenData, + ); + $this->assertInstanceOf(SlicedManifestGeneratorInterface::class, $generator); } }