diff --git a/packages/php-storage-driver-snowflake/provisioning/.gitignore b/packages/php-storage-driver-snowflake/provisioning/.gitignore new file mode 100644 index 000000000..7dcb7f631 --- /dev/null +++ b/packages/php-storage-driver-snowflake/provisioning/.gitignore @@ -0,0 +1,3 @@ +composer.lock +vendor +env_export diff --git a/packages/php-storage-driver-snowflake/provisioning/local/.gitignore b/packages/php-storage-driver-snowflake/provisioning/local/.gitignore new file mode 100644 index 000000000..18221f8eb --- /dev/null +++ b/packages/php-storage-driver-snowflake/provisioning/local/.gitignore @@ -0,0 +1,34 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log + +# Exclude all .tfvars files, which are likely to contain sentitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +# +*.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc diff --git a/packages/php-storage-driver-snowflake/provisioning/local/.terraform.lock.hcl b/packages/php-storage-driver-snowflake/provisioning/local/.terraform.lock.hcl new file mode 100644 index 000000000..c494f5ee4 --- /dev/null +++ b/packages/php-storage-driver-snowflake/provisioning/local/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "3.76.1" + constraints = "~> 3.74" + hashes = [ + "h1:5WSHHV9CgBvZ0rDDDxLnNHsjDfm4knb7ihJ2AIGB58A=", + "zh:1cf933104a641ffdb64d71a76806f4df35d19101b47e0eb02c9c36bd64bfdd2d", + "zh:273afaf908775ade6c9d32462938e7739ee8b00a0de2ef3cdddc5bc115bb1d4f", + "zh:2bc24ae989e38f575de034083082c69b41c54b8df69d35728853257c400ce0f4", + "zh:53ba88dbdaf9f818d35001c3d519a787f457283d9341f562dc3d0af51fd9606e", + "zh:5cdac7afea68bbd89d3bdb345d99470226482eff41f375f220fe338d2e5808da", + "zh:63127808890ac4be6cff6554985510b15ac715df698d550a3e722722dc56523c", + "zh:97a1237791f15373743189b078a0e0f2fa4dd7d7474077423376cd186312dc55", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a4f625e97e5f25073c08080e4a619f959bc0149fc853a6b1b49ab41d58b59665", + "zh:b56cca54019237941f7614e8d2712586a6ab3092e8e9492c70f06563259171e9", + "zh:d4bc33bfd6ac78fb61e6d48a61c179907dfdbdf149b89fb97272c663989a7fcd", + "zh:e0089d73fa56d128c574601305634a774eebacf4a84babba71da10040cecf99a", + "zh:e957531f1d92a6474c9b02bd9200da91b99ba07a0ab761c8e3176400dd41721c", + "zh:eceb85818d57d8270db4df7564cf4ed51b5c650a361aaa017c42227158e1946b", + "zh:f565e5caa1b349ec404c6d03d01c68b02233f5485ed038d0aab810dd4023a880", + ] +} diff --git a/packages/php-storage-driver-snowflake/provisioning/local/aws_s3.tf b/packages/php-storage-driver-snowflake/provisioning/local/aws_s3.tf new file mode 100644 index 000000000..52eafcf8a --- /dev/null +++ b/packages/php-storage-driver-snowflake/provisioning/local/aws_s3.tf @@ -0,0 +1,156 @@ +// File storage +resource "aws_s3_bucket" "S3FilesBucket" { + bucket = "${local.serviceName}-s3-files-storage-bucket" + + tags = { + Name = "keboola-file-storage" + } + + force_destroy = true +} + +resource "aws_s3_bucket_cors_configuration" "S3FilesBucketCorsConfiguration" { + bucket = aws_s3_bucket.S3FilesBucket.bucket + + cors_rule { + allowed_headers = ["*"] + allowed_methods = [ + "GET", + "PUT", + "POST", + "DELETE" + ] + allowed_origins = ["*"] + max_age_seconds = 3600 + } +} + +resource "aws_s3_bucket_lifecycle_configuration" "S3FilesBucketLifecycleConfig" { + bucket = aws_s3_bucket.S3FilesBucket.bucket + rule { + id = "After 30 days IA, 180 days to glacier and 270 delete" + filter { + prefix = "exp-180" + } + + expiration { + days = 270 + } + + transition { + storage_class = "STANDARD_IA" + days = 30 + } + + transition { + storage_class = "GLACIER" + days = 180 + } + + status = "Enabled" + } + + rule { + id = "Delete after 30 days" + status = "Enabled" + + filter { + prefix = "exp-30" + } + + expiration { + days = 30 + } + } + + rule { + id = "Delete after 15 days" + status = "Enabled" + + filter { + prefix = "exp-15" + } + + expiration { + days = 15 + } + } + + rule { + id = "Delete after 48 hours" + status = "Enabled" + + filter { + prefix = "exp-2" + } + expiration { + days = 2 + } + } + + rule { + id = "Delete incomplete multipart uploads" + status = "Enabled" + + abort_incomplete_multipart_upload { + days_after_initiation = 7 + } + } +} + +resource "aws_iam_user" "S3FileStorageUser" { + name = "${local.serviceName}-s3-file-storage-user" + + path = "/" +} + +data "aws_iam_policy_document" "S3AccessDocument" { + statement { + effect = "Allow" + actions = [ + "s3:*" + ] + resources = [ + "${aws_s3_bucket.S3FilesBucket.arn}/*" + ] + } + + statement { + sid = "AllowListingOfUserFolder" + actions = [ + "s3:ListBucket", + "s3:GetBucketLocation" + ] + effect = "Allow" + resources = [ + aws_s3_bucket.S3FilesBucket.arn + ] + } +} + +data "aws_iam_policy_document" "STSAccessDocument" { + statement { + effect = "Allow" + actions = [ + "sts:GetFederationToken" + ] + + resources = ["*"] + } +} + +resource "aws_iam_user_policy" "S3Access" { + name = "S3Access" + user = aws_iam_user.S3FileStorageUser.name + policy = data.aws_iam_policy_document.S3AccessDocument.json +} + +resource "aws_iam_user_policy" "STSAccess" { + name = "STSAccess" + user = aws_iam_user.S3FileStorageUser.name + policy = data.aws_iam_policy_document.STSAccessDocument.json +} + +resource "aws_iam_access_key" "S3FileStorageUserAccessKey" { + user = aws_iam_user.S3FileStorageUser.name +} diff --git a/packages/php-storage-driver-snowflake/provisioning/local/env-scripts/extract-variables-aws.sh b/packages/php-storage-driver-snowflake/provisioning/local/env-scripts/extract-variables-aws.sh new file mode 100755 index 000000000..fcacd9b41 --- /dev/null +++ b/packages/php-storage-driver-snowflake/provisioning/local/env-scripts/extract-variables-aws.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +TF_OUTPUTS_FILE=$1 + +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source ./functions.sh $TF_OUTPUTS_FILE + +# output variables +output_var 'AWS_S3_BUCKET' $(terraform_output 'FileStorageBucket') +output_var 'AWS_S3_KEY' 'exp-2' + +echo "" diff --git a/packages/php-storage-driver-snowflake/provisioning/local/env-scripts/extract-variables-common.sh b/packages/php-storage-driver-snowflake/provisioning/local/env-scripts/extract-variables-common.sh new file mode 100755 index 000000000..5b865fb3f --- /dev/null +++ b/packages/php-storage-driver-snowflake/provisioning/local/env-scripts/extract-variables-common.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +TF_OUTPUTS_FILE=$1 + +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source ./functions.sh $TF_OUTPUTS_FILE + +# output variables diff --git a/packages/php-storage-driver-snowflake/provisioning/local/env-scripts/functions.sh b/packages/php-storage-driver-snowflake/provisioning/local/env-scripts/functions.sh new file mode 100755 index 000000000..b3ffbf5cc --- /dev/null +++ b/packages/php-storage-driver-snowflake/provisioning/local/env-scripts/functions.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +TF_OUTPUTS_FILE=$1 +SCRIPT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${SCRIPT_PATH}/../../.." + +terraform_output() { + jq ".${1}.value" -r $TF_OUTPUTS_FILE +} + +terraform_output_json() { + jq ".${1}.value" -r $TF_OUTPUTS_FILE | jq -c +} + +output_var() { + echo "${1}=\"${2}\"" +} + +output_var_no_ticks() { + echo "${1}=${2}" +} + +output_var_json() { + echo "${1}='${2}'" +} + +output_file() { + mkdir -p "${PROJECT_ROOT}/$(dirname "${1}")" + echo "${2}" >"${PROJECT_ROOT}/${1}" +} diff --git a/packages/php-storage-driver-snowflake/provisioning/local/env-scripts/info/generate-actions-aws.sh b/packages/php-storage-driver-snowflake/provisioning/local/env-scripts/info/generate-actions-aws.sh new file mode 100755 index 000000000..08490d0e3 --- /dev/null +++ b/packages/php-storage-driver-snowflake/provisioning/local/env-scripts/info/generate-actions-aws.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +TF_OUTPUTS_FILE=$1 + +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source ./../functions.sh $TF_OUTPUTS_FILE diff --git a/packages/php-storage-driver-snowflake/provisioning/local/main.tf b/packages/php-storage-driver-snowflake/provisioning/local/main.tf new file mode 100644 index 000000000..4104e7d67 --- /dev/null +++ b/packages/php-storage-driver-snowflake/provisioning/local/main.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.74" + } + } +} + +variable "name_prefix" { + type = string +} + +locals { + serviceName = "${var.name_prefix}-php-storage-driver-snowflake" +} + +output "KEBOOLA_STORAGE_API__CLIENT_DB_PREFIX" { + value = var.name_prefix +} diff --git a/packages/php-storage-driver-snowflake/provisioning/local/update-env.sh b/packages/php-storage-driver-snowflake/provisioning/local/update-env.sh new file mode 100755 index 000000000..232e047eb --- /dev/null +++ b/packages/php-storage-driver-snowflake/provisioning/local/update-env.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ENV_FILE=".env" +INSERT_MODE=prepend +VERBOSE=false + +help () { + echo "Syntax: update-env.sh [-v] [-a] [-e ${ENV_FILE}] " + echo "Options:" + echo " -a|--append Append mode (used only when creating new env file, by default values are prepended to the env file)" + echo " -l|--localK8S Extract envs from local k8s cluster localed in kubernetes folder" + echo " -e|--env-file file Env file to write (default: ${ENV_FILE})" + echo " -v|--verbose Output extra information" + echo "" + echo "Example: update-env.sh aws" + echo "Example: update-env.sh -e .env.local azure" + echo "" +} + +LOCAL_K8S=false + +POSITIONAL_ARGS=() +while [[ $# -gt 0 ]]; do + case $1 in + -a|--append) + INSERT_MODE=append + shift + ;; + -e|--env-file) + ENV_FILE="$2" + shift + shift + ;; + -v|--verbose) + VERBOSE=true + shift + ;; + -l|--localK8S) + LOCAL_K8S=true + shift + ;; + -h|--help) + echo "Update env file with values from Terraform" + echo "" + help + exit 0 + ;; + -*|--*) + echo "Unknown option $1" + echo "" + help + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") + shift + ;; + esac +done +set -- "${POSITIONAL_ARGS[@]}" + +ENV_NAME=${1:-} +if [[ $ENV_NAME != "aws" && $ENV_NAME != "azure" && $ENV_NAME != "gcp" ]]; then + echo "Invalid environment name '${ENV_NAME}'. Possible values are: aws, azure, gcp" + echo "" + help + exit 1 +fi + +SCRIPT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${SCRIPT_PATH}/../.." +cd "${PROJECT_ROOT}" + +DELIMITER_START="##>> BEGIN GENERATED CONTENT <<##" +DELIMITER_END="##>> END GENERATED CONTENT <<##" + +if [ ! -f "${ENV_FILE}" ]; then + echo "Creating missing env file" + touch "${ENV_FILE}" +fi + +if [ "$LOCAL_K8S" = true ] ; then + SCRIPT_PATH="${SCRIPT_PATH}/kubernetes" + DELIMITER_START="##>> BEGIN K8S CONTENT <<##" + DELIMITER_END="##>> END K8S CONTENT <<##" + echo -e "Configuring \033[1;33m${ENV_FILE}\033[0m for local k8s cluster in folder \033[1;33m${SCRIPT_PATH}\033[0m" +else + echo -e "Configuring \033[1;33m${ENV_FILE}\033[0m for \033[1;33m${ENV_NAME}\033[0m in folder \033[1;33m${SCRIPT_PATH}\033[0m" +fi + +if ! grep -q "${DELIMITER_START}" "${ENV_FILE}"; then + if [[ $INSERT_MODE == "append" ]]; then + echo "Appending new auto-generated section to env file" + echo "" >> "${ENV_FILE}" + echo "${DELIMITER_START}" >> "${ENV_FILE}" + echo "${DELIMITER_END}" >> "${ENV_FILE}" + else + echo "Prepending new auto-generated section to env file" + ENV=$(cat "${ENV_FILE}") + echo "${DELIMITER_START}" > "${ENV_FILE}" + echo "${DELIMITER_END}" >> "${ENV_FILE}" + echo "" >> "${ENV_FILE}" + echo "${ENV}" >> "${ENV_FILE}" + fi +fi + +if [ "${VERBOSE}" = true ]; then + echo "Terraform outputs" + terraform -chdir="${SCRIPT_PATH}" output + echo "" +fi + +echo "Building variables" +ENV_TF_FILE="${ENV_FILE}.tf" +TF_OUTPUTS_FILE="${SCRIPT_PATH}/tfoutput.json" +trap "rm ${TF_OUTPUTS_FILE} || true; rm ${ENV_TF_FILE} || true" EXIT +terraform -chdir="${SCRIPT_PATH}" output -json > "${TF_OUTPUTS_FILE}" + +if [ "$LOCAL_K8S" = true ] ; then + "${SCRIPT_PATH}/extract-variables.sh" "$TF_OUTPUTS_FILE" "$PROJECT_ROOT" > "${ENV_TF_FILE}" +else + "${SCRIPT_PATH}/env-scripts/extract-variables-common.sh" "$TF_OUTPUTS_FILE" > "${ENV_TF_FILE}" + "${SCRIPT_PATH}/env-scripts/extract-variables-${ENV_NAME}.sh" "$TF_OUTPUTS_FILE" >> "${ENV_TF_FILE}" +fi + +echo "Writing variables" + +if [[ "$OSTYPE" == "darwin"* ]]; +then + sed -i '' -e "/${DELIMITER_START}/,/${DELIMITER_END}/{ /${DELIMITER_START}/{p; r ${ENV_TF_FILE} + }; /${DELIMITER_END}/p; d; }" "${ENV_FILE}"; +else + sed -i'' -e "/${DELIMITER_START}/,/${DELIMITER_END}/{ /${DELIMITER_START}/{p; r ${ENV_TF_FILE} + }; /${DELIMITER_END}/p; d; }" "${ENV_FILE}"; +fi + +echo "Done" + +if [ "$LOCAL_K8S" = false ] ; then + echo "Post update actions" + "${SCRIPT_PATH}/env-scripts/info/generate-actions-${ENV_NAME}.sh" "$TF_OUTPUTS_FILE" +fi diff --git a/packages/php-storage-driver-snowflake/tests/Functional/Handler/TableExportToFileHandlerTest.php b/packages/php-storage-driver-snowflake/tests/Functional/Handler/TableExportToFileHandlerTest.php new file mode 100644 index 000000000..0f1027f82 --- /dev/null +++ b/packages/php-storage-driver-snowflake/tests/Functional/Handler/TableExportToFileHandlerTest.php @@ -0,0 +1,194 @@ +createCredentialsWithKeyPair(); + + $handler = new TableExportToFileHandler(); + $command = (new TableExportToFileCommand()); + + $path = new RepeatedField(GPBType::STRING); + $path[] = self::SCHEMA_NAME; + + $command->setSource( + (new Table()) + ->setPath($path) + ->setTableName(self::TABLE_NAME), + ); + + $command->setFileProvider(FileProvider::S3); + $command->setFileFormat(FileFormat::CSV); + + // Create DestinationFile (similar to what's used in import/export) + $destination = new DestinationFile( + (string) getenv('AWS_ACCESS_KEY_ID'), + (string) getenv('AWS_SECRET_ACCESS_KEY'), + (string) getenv('AWS_REGION'), + (string) getenv('AWS_S3_BUCKET'), + $this->getExportDir() . '/workspace_export_test', + ); + + // Now create command parts from destination (reverse of TableExportToFileHandler::createDestination) + // Create S3Credentials from destination properties + $s3Credentials = (new S3Credentials()) + ->setKey($destination->getKey()) + ->setSecret($destination->getSecret()) + ->setRegion($destination->getRegion()); + + // Pack credentials into Any + $fileCredentials = new Any(); + $fileCredentials->pack($s3Credentials); + + // Split filePath into path and fileName + $filePath = $destination->getFilePath(); + $pathParts = explode('/', $filePath); + $fileName = array_pop($pathParts); + $pathWithoutFileName = implode('/', $pathParts); + + // Create FilePath + $command->setFilePath( + (new FilePath()) + ->setRoot($destination->getBucket()) + ->setPath($pathWithoutFileName) + ->setFileName($fileName), + ); + + $command->setFileCredentials($fileCredentials) + ->setExportOptions( + (new ExportOptions()) + ->setIsCompressed(false), + ); + + $response = $handler( + $credentials, + $command, + [], + new RuntimeOptions(), + ); + + $this->assertInstanceOf(TableExportToFileResponse::class, $response); + + } + + protected function setUp(): void + { + $this->getSnowflakeConnection(); + + $this->connection->executeQuery(sprintf( + 'CREATE OR REPLACE SCHEMA %s', + SnowflakeQuote::quoteSingleIdentifier(self::SCHEMA_NAME), + )); + + $this->connection->executeQuery( + sprintf( + 'DROP TABLE IF EXISTS %s.%s;', + SnowflakeQuote::quoteSingleIdentifier(self::SCHEMA_NAME), + SnowflakeQuote::quoteSingleIdentifier(self::TABLE_NAME), + ), + ); + + $this->connection->executeQuery( + (new SnowflakeTableQueryBuilder())->getCreateTableCommand( + self::SCHEMA_NAME, + self::TABLE_NAME, + new ColumnCollection([ + new SnowflakeColumn( + 'meal', + new Snowflake(Snowflake::TYPE_NUMBER), + ), + new SnowflakeColumn( + 'description', + new Snowflake(Snowflake::TYPE_VARCHAR), + ), + ]), + ), + ); + + $this->connection->executeQuery(sprintf( + // phpcs:disable + <<<'SQL' + INSERT INTO %s.%s ("meal", "description") VALUES + (7, 'Spaghetti with meatballs'), + (8, 'Chicken Alfredo'), + (9, 'Caesar Salad'), + (10, 'Grilled Salmon'); + SQL, + // phpcs:enable + SnowflakeQuote::quoteSingleIdentifier(self::SCHEMA_NAME), + SnowflakeQuote::quoteSingleIdentifier(self::TABLE_NAME), + )); + } + + protected function getDestinationSchema(): string + { + return self::SCHEMA_NAME; + } + + protected function getSourceSchema(): string + { + return self::SCHEMA_NAME; + } + + protected function getExportDir(): string + { + $buildPrefix = ''; + if (getenv('BUILD_PREFIX') !== false) { + $buildPrefix = getenv('BUILD_PREFIX'); + } + + return $this->getExportBlobDir() + . '-' + . $buildPrefix + . '-' + . getenv('SUITE'); + } + + protected function getExportBlobDir(): string + { + $path = ''; + switch (getenv('STORAGE_TYPE')) { +// case StorageType::STORAGE_S3: + case 'S3': + $key = getenv('AWS_S3_KEY'); + if ($key) { + $path = $key . '/'; + } + } + + return $path . 'test_export'; + } +}