keboola · zajca · Apr 25, 2024
diff --git a/packages/php-db-import-export/src/Backend/Bigquery/Export/GcsExportAdapter.php b/packages/php-db-import-export/src/Backend/Bigquery/Export/GcsExportAdapter.php
@@ -68,6 +68,7 @@ public function runCopyCommand(
         if ($exportOptions->generateManifest()) {
             (new Storage\GCS\ManifestGenerator\GcsSlicedManifestFromFolderGenerator(
                 $destination->getClient(),
+                new Storage\GCS\ManifestGenerator\WriteStreamFactory(),
             ))
                 ->generateAndSaveManifest($destination->getRelativePath());
         }

diff --git a/packages/php-db-import-export/src/Backend/Snowflake/Export/GcsExportAdapter.php b/packages/php-db-import-export/src/Backend/Snowflake/Export/GcsExportAdapter.php
@@ -9,6 +9,7 @@
 use Keboola\Db\ImportExport\ExportOptions;
 use Keboola\Db\ImportExport\ExportOptionsInterface;
 use Keboola\Db\ImportExport\Storage;
+use Keboola\Db\ImportExport\Storage\GCS\ManifestGenerator\WriteStreamFactory;
 
 class GcsExportAdapter implements BackendExportAdapterInterface
 {
@@ -73,6 +74,7 @@ public function runCopyCommand(
         if ($exportOptions->generateManifest()) {
             (new Storage\GCS\ManifestGenerator\GcsSlicedManifestFromUnloadQueryResultGenerator(
                 $destination->getClient(),
+                new WriteStreamFactory(),
             ))
                 ->generateAndSaveManifest($destination->getRelativePath(), $unloadedFiles);
         }

diff --git a/...-import-export/src/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromFolderGenerator.php b/...-import-export/src/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromFolderGenerator.php
@@ -6,7 +6,6 @@
 
 use Google\Cloud\Storage\StorageClient;
 use Google\Cloud\Storage\StorageObject;
-use Google\Cloud\Storage\WriteStream;
 use JsonException;
 use Keboola\Db\ImportExport\Storage\SlicedManifestGeneratorInterface;
 use Keboola\FileStorage\Path\RelativePathInterface;
@@ -16,12 +15,10 @@
  */
 class GcsSlicedManifestFromFolderGenerator implements SlicedManifestGeneratorInterface
 {
-    private StorageClient $client;
-
     public function __construct(
-        StorageClient $client,
+        private readonly StorageClient $client,
+        private readonly WriteStreamFactory $writeStreamFactory,
     ) {
-        $this->client = $client;
     }
 
     /**
@@ -46,9 +43,7 @@ public function generateAndSaveManifest(RelativePathInterface $path): void
             'entries' => $entries,
         ], JSON_THROW_ON_ERROR);
 
-        $writeStream = new WriteStream(null, [
-            'chunkSize' => self::CHUNK_SIZE_256_KB,
-        ]);
+        $writeStream = $this->writeStreamFactory->createWriteStream();
         $uploader = $bucket->getStreamableUploader($writeStream, [
             'name' => $path->getPathnameWithoutRoot() . 'manifest',
         ]);

diff --git a/...ort/src/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromUnloadQueryResultGenerator.php b/...ort/src/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromUnloadQueryResultGenerator.php
@@ -5,7 +5,6 @@
 namespace Keboola\Db\ImportExport\Storage\GCS\ManifestGenerator;
 
 use Google\Cloud\Storage\StorageClient;
-use Google\Cloud\Storage\WriteStream;
 use JsonException;
 use Keboola\Db\ImportExport\Storage\SlicedManifestGeneratorInterface;
 use Keboola\FileStorage\Path\RelativePathInterface;
@@ -15,11 +14,10 @@
  */
 class GcsSlicedManifestFromUnloadQueryResultGenerator implements SlicedManifestGeneratorInterface
 {
-    private StorageClient $gcsClient;
-
-    public function __construct(StorageClient $gcsClient)
-    {
-        $this->gcsClient = $gcsClient;
+    public function __construct(
+        private readonly StorageClient $gcsClient,
+        private readonly WriteStreamFactory $writeStreamFactory,
+    ) {
     }
 
     /**
@@ -39,9 +37,7 @@ public function generateAndSaveManifest(RelativePathInterface $path, array $list
         $manifest = [
             'entries' => $entries,
         ];
-        $writeStream = new WriteStream(null, [
-            'chunkSize' => self::CHUNK_SIZE_256_KB,
-        ]);
+        $writeStream = $this->writeStreamFactory->createWriteStream();
         $uploader = $bucket->getStreamableUploader($writeStream, [
             'name' => $path->getPathnameWithoutRoot() . 'manifest',
         ]);

diff --git a/packages/php-db-import-export/src/Storage/GCS/ManifestGenerator/WriteStreamFactory.php b/packages/php-db-import-export/src/Storage/GCS/ManifestGenerator/WriteStreamFactory.php
@@ -0,0 +1,18 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Keboola\Db\ImportExport\Storage\GCS\ManifestGenerator;
+
+use Google\Cloud\Storage\WriteStream;
+use Keboola\Db\ImportExport\Storage\SlicedManifestGeneratorInterface;
+
+class WriteStreamFactory
+{
+    public function createWriteStream(): WriteStream
+    {
+        return new WriteStream(null, [
+            'chunkSize' => SlicedManifestGeneratorInterface::CHUNK_SIZE_256_KB,
+        ]);
+    }
+}
diff --git a/packages/php-db-import-export/src/Storage/GCS/SnowflakeExportAdapter.php b/packages/php-db-import-export/src/Storage/GCS/SnowflakeExportAdapter.php
@@ -69,6 +69,7 @@ public function runCopyCommand(
         if ($exportOptions->generateManifest()) {
             (new Storage\GCS\ManifestGenerator\GcsSlicedManifestFromUnloadQueryResultGenerator(
                 $destination->getClient(),
+                new Storage\GCS\ManifestGenerator\WriteStreamFactory(),
             ))
                 ->generateAndSaveManifest($destination->getRelativePath(), $unloadedFiles);
         }

diff --git a/...ort/tests/unit/Storage/GCS/ManifestGenerator/GCSSlicedManifestFromFolderGeneratorTest.php b/...ort/tests/unit/Storage/GCS/ManifestGenerator/GCSSlicedManifestFromFolderGeneratorTest.php
@@ -10,7 +10,9 @@
 use Google\Cloud\Storage\ObjectIterator;
 use Google\Cloud\Storage\StorageClient;
 use Google\Cloud\Storage\StorageObject;
+use Google\Cloud\Storage\WriteStream;
 use Keboola\Db\ImportExport\Storage\GCS\ManifestGenerator\GcsSlicedManifestFromFolderGenerator;
+use Keboola\Db\ImportExport\Storage\GCS\ManifestGenerator\WriteStreamFactory;
 use Keboola\Db\ImportExport\Storage\SlicedManifestGeneratorInterface;
 use Keboola\FileStorage\Gcs\GcsProvider;
 use Keboola\FileStorage\Path\RelativePath;
@@ -21,6 +23,9 @@ class GCSSlicedManifestFromFolderGeneratorTest extends TestCase
 {
     public function testGenerateAndSaveManifest(): void
     {
+        $writeStreamMock = $this->createMock(WriteStream::class);
+        $writeStreamFactory = $this->createMock(WriteStreamFactory::class);
+        $writeStreamFactory->method('createWriteStream')->willReturn($writeStreamMock);
         $streamableUploaderMock = $this->getMockBuilder(StreamableUploader::class)
             ->disableOriginalConstructor()
             ->getMock();
@@ -47,10 +52,12 @@ public function testGenerateAndSaveManifest(): void
         $bucket->expects($this->exactly(2))
             ->method('name')
             ->willReturn('bucket1');
-
-        $bucket->expects($this->once())->method('getStreamableUploader')
-            ->willReturn($streamableUploaderMock);
-
+        $bucket->method('getStreamableUploader')->willReturnCallback(
+            function (WriteStream $w, array $o) use ($streamableUploaderMock) {
+                self::assertSame(['name' => 'prefix/xxxmanifest'], $o);
+                return $streamableUploaderMock;
+            },
+        );
         $clientMock = $this->createMock(StorageClient::class);
         $clientMock
             ->expects($this->once())
@@ -60,9 +67,21 @@ public function testGenerateAndSaveManifest(): void
 
         $path = RelativePath::createFromRootAndPath(new GcsProvider(), 'bucket1', 'prefix/xxx');
 
-        $generator = new GcsSlicedManifestFromFolderGenerator($clientMock);
+        $writtenData = '';
+        $writeStreamMock->method('write')->willReturnCallback(function (string $data) use (&$writtenData) {
+            $writtenData .= $data;
+            return 0;
+        });
+
+        $generator = new GcsSlicedManifestFromFolderGenerator($clientMock, $writeStreamFactory);
         $generator->generateAndSaveManifest($path);
 
+        $this->assertSame(
+        //phpcs:ignore
+            '{"entries":[{"url":"gs:\\/\\/bucket1\\/prefix\\/xxx\\/obj1_000000.csv","mandatory":true},{"url":"gs:\\/\\/bucket1\\/prefix\\/xxx\\/obj2_000000.csv","mandatory":true}]}',
+            $writtenData,
+        );
+
         $this->assertInstanceOf(SlicedManifestGeneratorInterface::class, $generator);
     }
 

diff --git a/...nit/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromUnloadQueryResultGeneratorTest.php b/...nit/Storage/GCS/ManifestGenerator/GcsSlicedManifestFromUnloadQueryResultGeneratorTest.php
@@ -7,49 +7,61 @@
 use Google\Cloud\Core\Upload\StreamableUploader;
 use Google\Cloud\Storage\Bucket;
 use Google\Cloud\Storage\StorageClient;
+use Google\Cloud\Storage\WriteStream;
 use Keboola\Db\ImportExport\Storage\GCS\ManifestGenerator\GcsSlicedManifestFromUnloadQueryResultGenerator;
+use Keboola\Db\ImportExport\Storage\GCS\ManifestGenerator\WriteStreamFactory;
 use Keboola\Db\ImportExport\Storage\SlicedManifestGeneratorInterface;
 use Keboola\FileStorage\Gcs\GcsProvider;
 use Keboola\FileStorage\Path\RelativePath;
-use PHPUnit\Framework\MockObject\MockObject;
 use PHPUnit\Framework\TestCase;
 
 class GcsSlicedManifestFromUnloadQueryResultGeneratorTest extends TestCase
 {
     public function testGenerateAndSaveManifest(): void
     {
-        $path = RelativePath::createFromRootAndPath(
-            new GcsProvider(),
-            'tomasfejfar-kbc-services-filestorag-s3filesbucket-ggrrgg35547q',
-            'permanent/256/snapshots/in/c-API-tests-e46793dac57ccf8cefb82ae9b8c05844cfabf985/languages/17982.csv.gz',
-        );
-
+        $writeStreamMock = $this->createMock(WriteStream::class);
+        $writeStreamFactory = $this->createMock(WriteStreamFactory::class);
+        $writeStreamFactory->method('createWriteStream')->willReturn($writeStreamMock);
         $streamableUploaderMock = $this->getMockBuilder(StreamableUploader::class)
             ->disableOriginalConstructor()
             ->getMock();
-        $streamableUploaderMock->method('getResumeUri')->willReturn('');
-        $streamableUploaderMock->method('upload')->willReturn([]);
 
-        $bucketMock = $this->getMockBuilder(Bucket::class)
-            ->disableOriginalConstructor()
-            ->setMethods(['upload', 'getStreamableUploader'])
-            ->getMock();
-        $bucketMock->expects($this->once())->method('getStreamableUploader')
-            ->willReturn($streamableUploaderMock);
+        $bucketMock = $this->createMock(Bucket::class);
+        $bucketMock->method('getStreamableUploader')->willReturnCallback(
+            function (WriteStream $w, array $o) use ($streamableUploaderMock) {
+                //phpcs:ignore
+                self::assertSame(['name' => 'permanent/256/snapshots/in/c-API-tests-e46793dac57ccf8cefb82ae9b8c05844cfabf985/languages/17982.csv.gzmanifest'], $o);
+                return $streamableUploaderMock;
+            },
+        );
 
-        /** @var MockObject|StorageClient $gcsClientMock */
-        $gcsClientMock = $this->getMockBuilder(StorageClient::class)
-            ->disableOriginalConstructor()
-            ->setMethods(['bucket'])
-            ->getMock();
+        $gcsClientMock = $this->createMock(StorageClient::class);
         $gcsClientMock->expects($this->once())->method('bucket')->willReturn($bucketMock);
 
-        $generator = new GcsSlicedManifestFromUnloadQueryResultGenerator($gcsClientMock);
+        $path = RelativePath::createFromRootAndPath(
+            new GcsProvider(),
+            'tomasfejfar-kbc-services-filestorag-s3filesbucket-ggrrgg35547q',
+            'permanent/256/snapshots/in/c-API-tests-e46793dac57ccf8cefb82ae9b8c05844cfabf985/languages/17982.csv.gz',
+        );
+
+        $writtenData = '';
+        $writeStreamMock->method('write')->willReturnCallback(function (string $data) use (&$writtenData) {
+            $writtenData .= $data;
+            return 0;
+        });
+
+        $generator = new GcsSlicedManifestFromUnloadQueryResultGenerator($gcsClientMock, $writeStreamFactory);
         $generator->generateAndSaveManifest($path, [
             ['FILE_NAME' => '17982.csv.gz_0_0_0.csv.gz', 'FILE_SIZE' => '10', 'ROW_COUNT' => '5'],
             ['FILE_NAME' => '17982.csv.gz_0_0_1.csv.gz', 'FILE_SIZE' => '25', 'ROW_COUNT' => '15'],
         ]);
 
+        $this->assertSame(
+        //phpcs:ignore
+            '{"entries":[{"url":"gs:\/\/tomasfejfar-kbc-services-filestorag-s3filesbucket-ggrrgg35547q\/permanent\/256\/snapshots\/in\/c-API-tests-e46793dac57ccf8cefb82ae9b8c05844cfabf985\/languages\/17982.csv.gz_0_0_0.csv.gz","mandatory":true},{"url":"gs:\/\/tomasfejfar-kbc-services-filestorag-s3filesbucket-ggrrgg35547q\/permanent\/256\/snapshots\/in\/c-API-tests-e46793dac57ccf8cefb82ae9b8c05844cfabf985\/languages\/17982.csv.gz_0_0_1.csv.gz","mandatory":true}]}',
+            $writtenData,
+        );
+
         $this->assertInstanceOf(SlicedManifestGeneratorInterface::class, $generator);
     }
 }