diff --git a/packages/php-db-import-export/phpstan-baseline.neon b/packages/php-db-import-export/phpstan-baseline.neon
index 11bcc0a9f..3c24ba975 100644
--- a/packages/php-db-import-export/phpstan-baseline.neon
+++ b/packages/php-db-import-export/phpstan-baseline.neon
@@ -1,5 +1,15 @@
parameters:
ignoreErrors:
+ -
+ message: "#^Method Keboola\\\\Db\\\\ImportExport\\\\Backend\\\\Bigquery\\\\BigqueryException\\:\\:createExceptionFromJobResult\\(\\) has parameter \\$jobInfo with no value type specified in iterable type array\\.$#"
+ count: 1
+ path: src/Backend/Bigquery/BigqueryException.php
+
+ -
+ message: "#^Method Keboola\\\\Db\\\\ImportExport\\\\Backend\\\\Bigquery\\\\BigqueryException\\:\\:getErrorMessageForErrorList\\(\\) has parameter \\$parsingErrors with no value type specified in iterable type array\\.$#"
+ count: 1
+ path: src/Backend/Bigquery/BigqueryException.php
+
-
message: "#^Cannot access offset 'durationSeconds' on mixed\\.$#"
count: 1
@@ -25,6 +35,11 @@ parameters:
count: 1
path: src/Backend/Snowflake/Importer.php
+ -
+ message: "#^Dead catch \\- Error is never thrown in the try block\\.$#"
+ count: 1
+ path: src/Backend/SourceDestinationColumnMap.php
+
-
message: "#^Method Keboola\\\\Db\\\\ImportExport\\\\Backend\\\\Synapse\\\\Exporter\\:\\:getAdapter\\(\\) should return Keboola\\\\Db\\\\ImportExport\\\\Backend\\\\Synapse\\\\SynapseExportAdapterInterface but returns object\\.$#"
count: 1
@@ -318,24 +333,4 @@ parameters:
-
message: "#^Variable \\$destination in PHPDoc tag @var does not match assigned variable \\$options\\.$#"
count: 1
- path: tests/unit/Storage/ABS/SynapseExportAdapterTest.php
-
- -
- message: "#^Method Keboola\\\\Db\\\\ImportExport\\\\Backend\\\\Bigquery\\\\BigqueryException::createExceptionFromJobResult\\(\\) has parameter \\$jobInfo with no value type specified in iterable type array\\.$#"
- count: 1
- path: src/Backend/Bigquery/BigqueryException.php
-
- -
- message: "#^Method Keboola\\\\Db\\\\ImportExport\\\\Backend\\\\Bigquery\\\\BigqueryException::getErrorMessageForErrorList\\(\\) has parameter \\$parsingErrors with no value type specified in iterable type array\\.$#"
- count: 1
- path: src/Backend/Bigquery/BigqueryException.php
-
- -
- message: "#^Dead catch - Error is never thrown in the try block\\.$#"
- count: 1
- path: src/Backend/SourceDestinationColumnMap.php
-
- -
- message: "#^Result of callable callable\\(T\\): void \\(void\\) is used\\.$#"
- count: 1
- path: tests/Common/StubLoader/GCSLoader.php
\ No newline at end of file
+ path: tests/unit/Storage/ABS/SynapseExportAdapterTest.php
\ No newline at end of file
diff --git a/packages/php-db-import-export/src/Backend/Bigquery/BigqueryImportOptions.php b/packages/php-db-import-export/src/Backend/Bigquery/BigqueryImportOptions.php
index a56e35f57..75df827a2 100644
--- a/packages/php-db-import-export/src/Backend/Bigquery/BigqueryImportOptions.php
+++ b/packages/php-db-import-export/src/Backend/Bigquery/BigqueryImportOptions.php
@@ -16,6 +16,7 @@ class BigqueryImportOptions extends ImportOptions
* @param self::USING_TYPES_* $usingTypes
* @param string[] $importAsNull
* @param string[] $features
+ * @param string[] $ignoreColumns
*/
public function __construct(
array $convertEmptyValuesToNull = [],
@@ -26,6 +27,7 @@ public function __construct(
?Session $session = null,
array $importAsNull = self::DEFAULT_IMPORT_AS_NULL,
array $features = [],
+ array $ignoreColumns = [],
) {
parent::__construct(
convertEmptyValuesToNull: $convertEmptyValuesToNull,
@@ -33,6 +35,7 @@ public function __construct(
useTimestamp: $useTimestamp,
numberOfIgnoredLines: $numberOfIgnoredLines,
usingTypes: $usingTypes,
+ ignoreColumns: $ignoreColumns,
importAsNull: $importAsNull,
features: $features,
);
diff --git a/packages/php-db-import-export/src/Backend/Bigquery/ToFinalTable/SqlBuilder.php b/packages/php-db-import-export/src/Backend/Bigquery/ToFinalTable/SqlBuilder.php
index 25fb53d5b..9a0490936 100644
--- a/packages/php-db-import-export/src/Backend/Bigquery/ToFinalTable/SqlBuilder.php
+++ b/packages/php-db-import-export/src/Backend/Bigquery/ToFinalTable/SqlBuilder.php
@@ -6,8 +6,8 @@
use Keboola\Datatype\Definition\BaseType;
use Keboola\Datatype\Definition\Bigquery;
-use Keboola\Db\Import\Exception;
use Keboola\Db\ImportExport\Backend\Bigquery\BigqueryImportOptions;
+use Keboola\Db\ImportExport\Backend\SourceDestinationColumnMap;
use Keboola\Db\ImportExport\Backend\ToStageImporterInterface;
use Keboola\TableBackendUtils\Column\Bigquery\BigqueryColumn;
use Keboola\TableBackendUtils\Escaping\Bigquery\BigqueryQuote;
@@ -17,33 +17,6 @@ class SqlBuilder
{
private const SRC_ALIAS = 'src';
- private function assertColumnExist(
- BigqueryTableDefinition $tableDefinition,
- BigqueryColumn $columnDefinition,
- ): BigqueryColumn {
- $destinationColumn = null;
- // case sensitive search
- /** @var BigqueryColumn $col */
- foreach ($tableDefinition->getColumnsDefinitions() as $col) {
- if ($col->getColumnName() === $columnDefinition->getColumnName()) {
- $destinationColumn = $col;
- break;
- }
- }
- if ($destinationColumn === null) {
- throw new Exception(
- sprintf(
- 'Columns "%s" can be imported as it was not found between columns "%s" of destination table.',
- $columnDefinition->getColumnName(),
- implode(', ', $tableDefinition->getColumnsNames()),
- ),
- Exception::UNKNOWN_ERROR,
- );
- }
-
- return $destinationColumn;
- }
-
public function getBeginTransaction(): string
{
return 'BEGIN TRANSACTION';
@@ -88,13 +61,13 @@ public function getDropTableIfExistsCommand(
*/
private function getColumnSetSqlPartForStringTable(
BigqueryTableDefinition $sourceTableDefinition,
- BigqueryTableDefinition $destinationTableDefinition,
+ SourceDestinationColumnMap $columnMap,
BigqueryImportOptions $importOptions,
): array {
$columnsSetSql = [];
/** @var BigqueryColumn $columnDefinition */
foreach ($sourceTableDefinition->getColumnsDefinitions() as $columnDefinition) {
- $destinationColumn = $this->assertColumnExist($destinationTableDefinition, $columnDefinition);
+ $destinationColumn = $columnMap->getDestination($columnDefinition);
if (in_array($columnDefinition->getColumnName(), $importOptions->getConvertEmptyValuesToNull(), true)) {
// use nullif only for string base type
if ($columnDefinition->getColumnDefinition()->getBasetype() === BaseType::STRING) {
@@ -146,6 +119,13 @@ public function getInsertAllIntoTargetTableCommand(
BigqueryImportOptions $importOptions,
string $timestamp,
): string {
+ $columnMap = SourceDestinationColumnMap::createForTables(
+ $sourceTableDefinition,
+ $destinationTableDefinition,
+ $importOptions->ignoreColumns(),
+ SourceDestinationColumnMap::MODE_MAP_BY_NAME,
+ );
+
$destinationTable = sprintf(
'%s.%s',
BigqueryQuote::quoteSingleIdentifier($destinationTableDefinition->getSchemaName()),
@@ -167,7 +147,7 @@ public function getInsertAllIntoTargetTableCommand(
$columnsSetSql = [];
/** @var BigqueryColumn $columnDefinition */
foreach ($sourceTableDefinition->getColumnsDefinitions() as $columnDefinition) {
- $this->assertColumnExist($destinationTableDefinition, $columnDefinition);
+ $columnMap->getDestination($columnDefinition);
$columnsSetSql[] = sprintf(
'%s.%s',
BigqueryQuote::quoteSingleIdentifier(self::SRC_ALIAS),
@@ -177,7 +157,7 @@ public function getInsertAllIntoTargetTableCommand(
} else {
$columnsSetSql = $this->getColumnSetSqlPartForStringTable(
$sourceTableDefinition,
- $destinationTableDefinition,
+ $columnMap,
$importOptions,
);
}
@@ -285,28 +265,37 @@ public function getUpdateWithPkCommand(
): string {
$columnsSet = [];
- foreach ($stagingTableDefinition->getColumnsNames() as $columnName) {
+ $columnMap = SourceDestinationColumnMap::createForTables(
+ $stagingTableDefinition,
+ $destinationTableDefinition,
+ $importOptions->ignoreColumns(),
+ SourceDestinationColumnMap::MODE_MAP_BY_NAME,
+ );
+
+ foreach ($stagingTableDefinition->getColumnsDefinitions() as $sourceColumn) {
+ $destinationColumn = $columnMap->getDestination($sourceColumn);
+
if ($importOptions->usingUserDefinedTypes()) {
$columnsSet[] = sprintf(
'%s = `src`.%s',
- BigqueryQuote::quoteSingleIdentifier($columnName),
- BigqueryQuote::quoteSingleIdentifier($columnName),
+ BigqueryQuote::quoteSingleIdentifier($destinationColumn->getColumnName()),
+ BigqueryQuote::quoteSingleIdentifier($sourceColumn->getColumnName()),
);
continue;
}
// if string table convert nulls<=>''
- if (in_array($columnName, $importOptions->getConvertEmptyValuesToNull(), true)) {
+ if (in_array($sourceColumn->getColumnName(), $importOptions->getConvertEmptyValuesToNull(), true)) {
$columnsSet[] = sprintf(
'%s = IF(`src`.%s = \'\', NULL, `src`.%s)',
- BigqueryQuote::quoteSingleIdentifier($columnName),
- BigqueryQuote::quoteSingleIdentifier($columnName),
- BigqueryQuote::quoteSingleIdentifier($columnName),
+ BigqueryQuote::quoteSingleIdentifier($destinationColumn->getColumnName()),
+ BigqueryQuote::quoteSingleIdentifier($sourceColumn->getColumnName()),
+ BigqueryQuote::quoteSingleIdentifier($sourceColumn->getColumnName()),
);
} else {
$columnsSet[] = sprintf(
'%s = COALESCE(`src`.%s, \'\')',
- BigqueryQuote::quoteSingleIdentifier($columnName),
- BigqueryQuote::quoteSingleIdentifier($columnName),
+ BigqueryQuote::quoteSingleIdentifier($destinationColumn->getColumnName()),
+ BigqueryQuote::quoteSingleIdentifier($sourceColumn->getColumnName()),
);
}
}
@@ -321,16 +310,14 @@ public function getUpdateWithPkCommand(
$columnsComparisonSql = [];
if ($importOptions->compareAllColumnsInNativeTable()) {
- $columnsComparisonSql = array_map(
- static function ($columnName) {
- return sprintf(
- '`dest`.%s IS DISTINCT FROM `src`.%s',
- BigqueryQuote::quoteSingleIdentifier($columnName),
- BigqueryQuote::quoteSingleIdentifier($columnName),
- );
- },
- $stagingTableDefinition->getColumnsNames(),
- );
+ foreach ($stagingTableDefinition->getColumnsDefinitions() as $sourceColumn) {
+ $destinationColumn = $columnMap->getDestination($sourceColumn);
+ $columnsComparisonSql[] = sprintf(
+ '`dest`.%s IS DISTINCT FROM `src`.%s',
+ BigqueryQuote::quoteSingleIdentifier($destinationColumn->getColumnName()),
+ BigqueryQuote::quoteSingleIdentifier($sourceColumn->getColumnName()),
+ );
+ }
}
$dest = sprintf(
diff --git a/packages/php-db-import-export/src/Backend/Bigquery/ToStage/FromTableInsertIntoAdapter.php b/packages/php-db-import-export/src/Backend/Bigquery/ToStage/FromTableInsertIntoAdapter.php
index 65499ee86..5741623f6 100644
--- a/packages/php-db-import-export/src/Backend/Bigquery/ToStage/FromTableInsertIntoAdapter.php
+++ b/packages/php-db-import-export/src/Backend/Bigquery/ToStage/FromTableInsertIntoAdapter.php
@@ -51,6 +51,7 @@ public function runCopyCommand(
$source->getTableName(),
))->getColumnsDefinitions(),
destination: $destination->getColumnsDefinitions(),
+ ignoreSourceColumns: $importOptions->ignoreColumns(),
assertOptions: Assert::ASSERT_MINIMAL,
);
}
diff --git a/packages/php-db-import-export/tests/Common/StubLoader/GCSLoader.php b/packages/php-db-import-export/tests/Common/StubLoader/GCSLoader.php
index 0970ee26e..d07a11ea5 100644
--- a/packages/php-db-import-export/tests/Common/StubLoader/GCSLoader.php
+++ b/packages/php-db-import-export/tests/Common/StubLoader/GCSLoader.php
@@ -113,7 +113,7 @@ public function load(): void
echo PHP_EOL;
$promises = [];
- $promises[] = fn() => new Promise(function ($resolve) use ($bucket) {
+ $promises[] = fn() => new Promise(function (callable $resolve) use ($bucket): void {
$blobName = '02_tw_accounts.csv.invalid.manifest';
$res = $bucket->upload(
json_encode([
@@ -131,14 +131,16 @@ public function load(): void
'name' => $blobName,
],
);
- return $resolve([$blobName, $res]);
+ $resolve([$blobName, $res]);
});
- parallel($promises)->then(function (): void {
- return;
- }, function (Throwable $e): void {
- echo 'Error: ' . $e->getMessage() . PHP_EOL;
- });
+ parallel($promises)->then(
+ function (): void {
+ },
+ function (Throwable $e): void {
+ echo 'Error: ' . $e->getMessage() . PHP_EOL;
+ },
+ );
echo "GCS load complete \n";
}
diff --git a/packages/php-storage-driver-common/generated/GPBMetadata/Proto/Table.php b/packages/php-storage-driver-common/generated/GPBMetadata/Proto/Table.php
index 9df4f9dde..c07053207 100644
Binary files a/packages/php-storage-driver-common/generated/GPBMetadata/Proto/Table.php and b/packages/php-storage-driver-common/generated/GPBMetadata/Proto/Table.php differ
diff --git a/packages/php-storage-driver-common/generated/Keboola/StorageDriver/Command/Table/ImportExportShared/ImportOptions.php b/packages/php-storage-driver-common/generated/Keboola/StorageDriver/Command/Table/ImportExportShared/ImportOptions.php
index f368acea7..43e176f85 100644
--- a/packages/php-storage-driver-common/generated/Keboola/StorageDriver/Command/Table/ImportExportShared/ImportOptions.php
+++ b/packages/php-storage-driver-common/generated/Keboola/StorageDriver/Command/Table/ImportExportShared/ImportOptions.php
@@ -68,6 +68,12 @@ class ImportOptions extends \Google\Protobuf\Internal\Message
* Generated from protobuf field repeated string importAsNull = 9;
*/
private $importAsNull;
+ /**
+ * columns that are ignored during import, used to ignore _timestamp column from check when importing table from workspace
+ *
+ * Generated from protobuf field repeated string ignoreColumnsNames = 10;
+ */
+ private $ignoreColumnsNames;
/**
* Constructor.
@@ -93,6 +99,8 @@ class ImportOptions extends \Google\Protobuf\Internal\Message
* @type int $createMode
* @type array|\Google\Protobuf\Internal\RepeatedField $importAsNull
* list of values which should be imported as null, default should be always empty string
+ * @type array|\Google\Protobuf\Internal\RepeatedField $ignoreColumnsNames
+ * columns that are ignored during import, used to ignore _timestamp column from check when importing table from workspace
* }
*/
public function __construct($data = NULL) {
@@ -330,6 +338,32 @@ public function setImportAsNull($var)
return $this;
}
+ /**
+ * columns that are ignored during import, used to ignore _timestamp column from check when importing table from workspace
+ *
+ * Generated from protobuf field repeated string ignoreColumnsNames = 10;
+ * @return \Google\Protobuf\Internal\RepeatedField
+ */
+ public function getIgnoreColumnsNames()
+ {
+ return $this->ignoreColumnsNames;
+ }
+
+ /**
+ * columns that are ignored during import, used to ignore _timestamp column from check when importing table from workspace
+ *
+ * Generated from protobuf field repeated string ignoreColumnsNames = 10;
+ * @param array|\Google\Protobuf\Internal\RepeatedField $var
+ * @return $this
+ */
+ public function setIgnoreColumnsNames($var)
+ {
+ $arr = GPBUtil::checkRepeatedField($var, \Google\Protobuf\Internal\GPBType::STRING);
+ $this->ignoreColumnsNames = $arr;
+
+ return $this;
+ }
+
}
// Adding a class alias for backwards compatibility with the previous class name.
diff --git a/packages/php-storage-driver-common/proto/table.proto b/packages/php-storage-driver-common/proto/table.proto
index ddbab8e9b..d9af3d0cf 100644
--- a/packages/php-storage-driver-common/proto/table.proto
+++ b/packages/php-storage-driver-common/proto/table.proto
@@ -210,6 +210,8 @@ message ImportExportShared {
CreateMode createMode = 8;
repeated string importAsNull = 9; // list of values which should be imported as null, default should be always empty string
+
+ repeated string ignoreColumnsNames = 10; // columns that are ignored during import, used to ignore _timestamp column from check when importing table from workspace
}
/**
* Common export options