From 15d1a2c9625bddad599b0b503359d6e67f5dce73 Mon Sep 17 00:00:00 2001 From: Katsuhiko YOSHIDA Date: Sun, 4 Apr 2021 01:10:25 +0900 Subject: [PATCH 01/28] Add description of tables support --- lib/embulk/output/bigquery/bigquery_client.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/embulk/output/bigquery/bigquery_client.rb b/lib/embulk/output/bigquery/bigquery_client.rb index 13753d2..e956291 100644 --- a/lib/embulk/output/bigquery/bigquery_client.rb +++ b/lib/embulk/output/bigquery/bigquery_client.rb @@ -419,6 +419,7 @@ def create_table_if_not_exists(table, dataset: nil, options: nil) table_reference: { table_id: table, }, + description: @task['description'], schema: { fields: fields, } From 06a64767363dae525a8e1661494bda9d0b883729 Mon Sep 17 00:00:00 2001 From: Katsuhiko YOSHIDA Date: Sun, 4 Apr 2021 01:13:37 +0900 Subject: [PATCH 02/28] Add description of columns support --- lib/embulk/output/bigquery/helper.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/embulk/output/bigquery/helper.rb b/lib/embulk/output/bigquery/helper.rb index f0b41b3..37cac69 100644 --- a/lib/embulk/output/bigquery/helper.rb +++ b/lib/embulk/output/bigquery/helper.rb @@ -46,10 +46,11 @@ def self.fields_from_embulk_schema(task, schema) embulk_type = column[:type] column_option = column_options_map[column_name] || {} {}.tap do |field| - field[:name] = column_name - field[:type] = (column_option['type'] || bq_type_from_embulk_type(embulk_type)).upcase - field[:mode] = column_option['mode'] if column_option['mode'] - field[:fields] = deep_symbolize_keys(column_option['fields']) if column_option['fields'] + field[:name] = column_name + field[:type] = (column_option['type'] || bq_type_from_embulk_type(embulk_type)).upcase + field[:mode] = column_option['mode'] if column_option['mode'] + field[:fields] = deep_symbolize_keys(column_option['fields']) if column_option['fields'] + field[:description] = column_option['description'] if column_option['description'] end end end From bd2a74814750ddc1d5d32606bf7f31b0cdbe3d36 Mon Sep 17 00:00:00 2001 From: Katsuhiko YOSHIDA Date: Sun, 4 Apr 2021 01:29:53 +0900 Subject: [PATCH 03/28] Update document --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b36847b..c592845 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ OAuth flow for installed applications. | gcs_bucket | string | optional | nil | See [GCS Bucket](#gcs-bucket) | | auto_create_gcs_bucket | boolean | optional | false | See [GCS Bucket](#gcs-bucket) | | progress_log_interval | float | optional | nil (Disabled) | Progress log interval. The progress log is disabled by nil (default). NOTE: This option may be removed in a future because a filter plugin can achieve the same goal | +| description | string | optional | nil | description of table | Client or request options @@ -316,6 +317,7 @@ Column options are used to aid guessing BigQuery schema, or to define conversion - json: `STRING`, `RECORD` (default: `STRING`) - **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`) - **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column. + - **description**: description (string, default is `None`). - **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is `default_timestamp_format`) - **timezone**: timezone to convert into/from `timestamp`, `date` (string, default is `default_timezone`). - **default_timestamp_format**: default timestamp format for column_options (string, default is "%Y-%m-%d %H:%M:%S.%6N") From 1ebfcace4da06726b6386e1372aca6e1465207fc Mon Sep 17 00:00:00 2001 From: fagai Date: Tue, 18 Jan 2022 22:30:08 +0900 Subject: [PATCH 04/28] fix test --- test/test_transaction.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_transaction.rb b/test/test_transaction.rb index 575d0ca..196715a 100644 --- a/test/test_transaction.rb +++ b/test/test_transaction.rb @@ -109,7 +109,7 @@ def test_replace task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE') mock(obj).delete_table(config['temp_table']) @@ -122,7 +122,7 @@ def test_replace_with_partitioning task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE') mock(obj).delete_table(config['temp_table']) @@ -138,7 +138,7 @@ def test_replace_backup any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) mock(obj).get_dataset(config['dataset_old']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table']) mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old']) @@ -158,7 +158,7 @@ def test_replace_backup_auto_create_dataset mock(obj).create_dataset(config['dataset']) mock(obj).create_dataset(config['dataset_old'], reference: config['dataset']) mock(obj).create_table_if_not_exists(config['table']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old']) mock(obj).get_table_or_partition(config['table']) @@ -176,7 +176,7 @@ def test_replace_backup_with_partitioning any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) mock(obj).get_dataset(config['dataset_old']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table']) mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old']) @@ -196,7 +196,7 @@ def test_append task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND') mock(obj).delete_table(config['temp_table']) @@ -209,7 +209,7 @@ def test_append_with_partitioning task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND') mock(obj).delete_table(config['temp_table']) From 72372d1f1d4c7ebfae2471275cc43bbf97375891 Mon Sep 17 00:00:00 2001 From: fagai Date: Tue, 18 Jan 2022 22:30:30 +0900 Subject: [PATCH 05/28] add description test --- test/test_helper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_helper.rb b/test/test_helper.rb index 11f30eb..17cb1c7 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -68,7 +68,7 @@ def test_fields_from_embulk_schema_with_column_options ]) task = { 'column_options' => [ - {'name' => 'boolean', 'type' => 'STRING', 'mode' => 'REQUIRED'}, + {'name' => 'boolean', 'type' => 'STRING', 'mode' => 'REQUIRED', 'description' => 'hoge'}, {'name' => 'long', 'type' => 'STRING'}, {'name' => 'double', 'type' => 'STRING'}, {'name' => 'string', 'type' => 'INTEGER'}, @@ -81,7 +81,7 @@ def test_fields_from_embulk_schema_with_column_options ], } expected = [ - {name: 'boolean', type: 'STRING', mode: 'REQUIRED'}, + {name: 'boolean', type: 'STRING', mode: 'REQUIRED', description: 'hoge'}, {name: 'long', type: 'STRING'}, {name: 'double', type: 'STRING'}, {name: 'string', type: 'INTEGER'}, From ac94f9abb9f303243fd16383d7736f4a01ff7e41 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sun, 23 Apr 2023 11:09:44 +0900 Subject: [PATCH 06/28] Add GitHub Actions --- .github/workflows/check.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/check.yml diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..af1c9ea --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,33 @@ +name: Check +on: [ pull_request, push ] +jobs: + check: + runs-on: ubuntu-latest + # push: always run. + # pull_request: run only when the PR is submitted from a forked repository, not within this repository. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v3 + - name: Set up OpenJDK 8 + uses: actions/setup-java@v3 + with: + java-version: 8 + distribution: "temurin" + - uses: ruby/setup-ruby@v1 + with: + ruby-version: 'jruby-9.1.17.0' # 9.1.15.0 doesn't available in setup-ruby@v1 + bundler-cache: true + - name: show ruby version + run: ruby -v + # Gem::LoadError: You have already activated rake 10.4.2, + # but your Gemfile requires rake 13.0.6. + - name: gem install rake -v 13.0.6 + run: gem install rake -v 13.0.6 + - name: bundle install + run: bundle install + - name: install embulk + run: "curl -L -o embulk.jar https://github.com/embulk/embulk/releases/download/v0.8.39/embulk-0.8.39.jar" + - name: rake test + run: bundle exec env RUBYOPT="-r ./embulk.jar" rake test From 9dc252adf4d256f7ae6f5e83de20c0aece5f3efd Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sun, 23 Apr 2023 13:56:08 +0900 Subject: [PATCH 07/28] Fix unexpected method invocation --- test/test_transaction.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_transaction.rb b/test/test_transaction.rb index 575d0ca..cba7db1 100644 --- a/test/test_transaction.rb +++ b/test/test_transaction.rb @@ -109,7 +109,7 @@ def test_replace task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], {:options=>{"expiration_time"=>nil}}) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE') mock(obj).delete_table(config['temp_table']) @@ -122,7 +122,7 @@ def test_replace_with_partitioning task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'],{:options=>{"expiration_time"=>nil}}) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE') mock(obj).delete_table(config['temp_table']) @@ -138,7 +138,7 @@ def test_replace_backup any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) mock(obj).get_dataset(config['dataset_old']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], {:options=>{"expiration_time"=>nil}}) mock(obj).create_table_if_not_exists(config['table']) mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old']) @@ -158,7 +158,7 @@ def test_replace_backup_auto_create_dataset mock(obj).create_dataset(config['dataset']) mock(obj).create_dataset(config['dataset_old'], reference: config['dataset']) mock(obj).create_table_if_not_exists(config['table']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'],{:options=>{"expiration_time"=>nil}}) mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old']) mock(obj).get_table_or_partition(config['table']) @@ -176,7 +176,7 @@ def test_replace_backup_with_partitioning any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) mock(obj).get_dataset(config['dataset_old']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'],{:options=>{"expiration_time"=>nil}}) mock(obj).create_table_if_not_exists(config['table']) mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old']) @@ -196,7 +196,7 @@ def test_append task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], {:options=>{"expiration_time"=>nil}}) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND') mock(obj).delete_table(config['temp_table']) @@ -209,7 +209,7 @@ def test_append_with_partitioning task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], {:options=>{"expiration_time"=>nil}}) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND') mock(obj).delete_table(config['temp_table']) From 7d82eb191b626685c78b283120a46d6a51933dad Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sun, 23 Apr 2023 17:50:05 +0900 Subject: [PATCH 08/28] Update gemspec --- Gemfile | 3 +-- embulk-output-bigquery.gemspec | 17 ++++++----------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/Gemfile b/Gemfile index 7328ea6..824ce0e 100644 --- a/Gemfile +++ b/Gemfile @@ -1,8 +1,7 @@ source 'https://rubygems.org/' gemspec -gem 'embulk', '< 0.10' -gem 'liquid', '= 4.0.0' # the version included in embulk.jar +gem 'embulk', '> 0.10.0' gem 'embulk-parser-none' gem 'embulk-parser-jsonl' gem 'pry-nav' diff --git a/embulk-output-bigquery.gemspec b/embulk-output-bigquery.gemspec index 14fae7e..df13d27 100644 --- a/embulk-output-bigquery.gemspec +++ b/embulk-output-bigquery.gemspec @@ -14,18 +14,13 @@ Gem::Specification.new do |spec| spec.test_files = spec.files.grep(%r{^(test|spec)/}) spec.require_paths = ["lib"] - # TODO - # signet 0.12.0 and google-api-client 0.33.0 require >= Ruby 2.4. - # Embulk 0.9 use JRuby 9.1.X.Y and it's compatible with Ruby 2.3. - # So, force install signet < 0.12 and google-api-client < 0.33.0 - # Also, representable version >= 3.1.0 requires Ruby version >= 2.4 - spec.add_dependency 'signet', '~> 0.7', '< 0.12.0' - spec.add_dependency 'google-api-client','< 0.33.0' + # the latest version + spec.add_dependency 'google-api-client','= 0.53.0' spec.add_dependency 'time_with_zone' - spec.add_dependency "representable", ['~> 3.0.0', '< 3.1'] - # faraday 1.1.0 require >= Ruby 2.4. - # googleauth 0.9.0 requires faraday ~> 0.12 - spec.add_dependency "faraday", '~> 0.12' + spec.add_dependency 'thwait' + # activesupport require Ruby >= 2.7.0 + # jruby-9.3.0.0 is MRI 2.6 compatible + spec.add_dependency 'activesupport', "< 7.0" spec.add_development_dependency 'bundler', ['>= 1.10.6'] spec.add_development_dependency 'rake', ['>= 10.0'] From 2eca01ddac8a3993550e73ca2bf4deef145e6aa9 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sat, 15 Apr 2023 23:07:00 +0900 Subject: [PATCH 09/28] Use double splat JRuby 9.4.x compatible MRI 3.1. MRI 3.0 introduced "REAL" keyward argument. https://bugs.ruby-lang.org/issues/14183 This codes raise Argument Error. def foo(a, b, c, key: 1) end h = {key: 42} foo(1, 2, 3, h) It is necessary to change like the following. def foo(a, b, c, key: 1) end h = {key: 42} foo(1, 2, 3, **h) --- lib/embulk/output/bigquery/bigquery_client.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/embulk/output/bigquery/bigquery_client.rb b/lib/embulk/output/bigquery/bigquery_client.rb index 5ff20f5..4e0750b 100644 --- a/lib/embulk/output/bigquery/bigquery_client.rb +++ b/lib/embulk/output/bigquery/bigquery_client.rb @@ -121,7 +121,7 @@ def load_from_gcs(object_uris, table) opts = {} Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" } - response = with_network_retry { client.insert_job(@project, body, opts) } + response = with_network_retry { client.insert_job(@project, body, **opts) } unless @task['is_skip_job_result_check'] response = wait_load('Load', response) end @@ -222,7 +222,7 @@ def load(path, table, write_disposition: 'WRITE_APPEND') # }, } Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" } - response = with_network_retry { client.insert_job(@project, body, opts) } + response = with_network_retry { client.insert_job(@project, body, **opts) } if @task['is_skip_job_result_check'] response else @@ -278,7 +278,7 @@ def copy(source_table, destination_table, destination_dataset = nil, write_dispo opts = {} Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" } - response = with_network_retry { client.insert_job(@project, body, opts) } + response = with_network_retry { client.insert_job(@project, body, **opts) } wait_load('Copy', response) rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e response = {status_code: e.status_code, message: e.message, error_class: e.class} @@ -372,7 +372,7 @@ def create_dataset(dataset = nil, reference: nil) end opts = {} Embulk.logger.debug { "embulk-output-bigquery: insert_dataset(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" } - with_network_retry { client.insert_dataset(@project, body, opts) } + with_network_retry { client.insert_dataset(@project, body, **opts) } rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e if e.status_code == 409 && /Already Exists:/ =~ e.message # ignore 'Already Exists' error @@ -447,7 +447,7 @@ def create_table_if_not_exists(table, dataset: nil, options: nil) opts = {} Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" } - with_network_retry { client.insert_table(@project, dataset, body, opts) } + with_network_retry { client.insert_table(@project, dataset, body, **opts) } rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e if e.status_code == 409 && /Already Exists:/ =~ e.message # ignore 'Already Exists' error From 067ff15e00fbe2ba6742aedd6526616e1e438513 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sun, 23 Apr 2023 18:08:35 +0900 Subject: [PATCH 10/28] Use new Embulk bootstrap mechanism. --- test/helper.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/helper.rb b/test/helper.rb index 4b38d7d..906125d 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -4,14 +4,14 @@ require 'test/unit' require 'test/unit/rr' +# Embulk 0.10.x introduced new bootstrap mechanism. +# https://github.com/embulk/embulk/blob/641f35fec064cca7b1a7314d634a4b64ef8637f1/embulk-ruby/test/vanilla/run-test.rb#L8-L13 +static_initializer = Java::org.embulk.EmbulkDependencyClassLoader.staticInitializer().useSelfContainedJarFiles() +static_initializer.java_send :initialize + +require 'embulk/java/bootstrap' require 'embulk' -begin - # Embulk ~> 0.8.x - Embulk.setup -rescue NotImplementedError, NoMethodError, NameError - # Embulk ~> 0.9.x - require 'embulk/java/bootstrap' -end + Embulk.logger = Embulk::Logger.new('/dev/null') APP_ROOT = File.expand_path('../', __dir__) From b8ae870613cf63f2e4c680b297f12dfd8f044555 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sun, 23 Apr 2023 18:13:33 +0900 Subject: [PATCH 11/28] Update GitHub Actions --- .github/workflows/check.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index af1c9ea..e878e98 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -17,17 +17,13 @@ jobs: distribution: "temurin" - uses: ruby/setup-ruby@v1 with: - ruby-version: 'jruby-9.1.17.0' # 9.1.15.0 doesn't available in setup-ruby@v1 + ruby-version: 'jruby-9.3.10.0' bundler-cache: true - name: show ruby version run: ruby -v - # Gem::LoadError: You have already activated rake 10.4.2, - # but your Gemfile requires rake 13.0.6. - - name: gem install rake -v 13.0.6 - run: gem install rake -v 13.0.6 - name: bundle install run: bundle install - - name: install embulk - run: "curl -L -o embulk.jar https://github.com/embulk/embulk/releases/download/v0.8.39/embulk-0.8.39.jar" + - name: install embulk.jar + run: "curl -L -o embulk.jar https://github.com/embulk/embulk/releases/download/v0.10.49/embulk-0.10.49.jar" - name: rake test run: bundle exec env RUBYOPT="-r ./embulk.jar" rake test From 3fa1871637fd36db175a4971d5553515dedd04c3 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sun, 23 Apr 2023 18:27:05 +0900 Subject: [PATCH 12/28] Fix unexpected method invocation for REAL keyward argument --- test/test_transaction.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_transaction.rb b/test/test_transaction.rb index cba7db1..5f8b972 100644 --- a/test/test_transaction.rb +++ b/test/test_transaction.rb @@ -109,7 +109,7 @@ def test_replace task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table'], {:options=>{"expiration_time"=>nil}}) + mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil}) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE') mock(obj).delete_table(config['temp_table']) @@ -122,7 +122,7 @@ def test_replace_with_partitioning task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table'],{:options=>{"expiration_time"=>nil}}) + mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil}) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE') mock(obj).delete_table(config['temp_table']) @@ -138,7 +138,7 @@ def test_replace_backup any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) mock(obj).get_dataset(config['dataset_old']) - mock(obj).create_table_if_not_exists(config['temp_table'], {:options=>{"expiration_time"=>nil}}) + mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil}) mock(obj).create_table_if_not_exists(config['table']) mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old']) @@ -158,7 +158,7 @@ def test_replace_backup_auto_create_dataset mock(obj).create_dataset(config['dataset']) mock(obj).create_dataset(config['dataset_old'], reference: config['dataset']) mock(obj).create_table_if_not_exists(config['table']) - mock(obj).create_table_if_not_exists(config['temp_table'],{:options=>{"expiration_time"=>nil}}) + mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil}) mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old']) mock(obj).get_table_or_partition(config['table']) @@ -176,7 +176,7 @@ def test_replace_backup_with_partitioning any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) mock(obj).get_dataset(config['dataset_old']) - mock(obj).create_table_if_not_exists(config['temp_table'],{:options=>{"expiration_time"=>nil}}) + mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil}) mock(obj).create_table_if_not_exists(config['table']) mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old']) @@ -196,7 +196,7 @@ def test_append task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table'], {:options=>{"expiration_time"=>nil}}) + mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil}) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND') mock(obj).delete_table(config['temp_table']) @@ -209,7 +209,7 @@ def test_append_with_partitioning task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table'], {:options=>{"expiration_time"=>nil}}) + mock(obj).create_table_if_not_exists(config['temp_table'], options: {"expiration_time"=>nil}) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND') mock(obj).delete_table(config['temp_table']) From 4519c262118d35a4ac27b14fa29d1072423cdab0 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sun, 23 Apr 2023 18:32:30 +0900 Subject: [PATCH 13/28] Add matrix tests --- .github/workflows/check.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index e878e98..9b85ab9 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -7,6 +7,10 @@ jobs: # pull_request: run only when the PR is submitted from a forked repository, not within this repository. if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository strategy: + matrix: + jruby_version: + - 9.3.10.0 + - 9.4.2.0 fail-fast: false steps: - uses: actions/checkout@v3 @@ -17,7 +21,7 @@ jobs: distribution: "temurin" - uses: ruby/setup-ruby@v1 with: - ruby-version: 'jruby-9.3.10.0' + ruby-version: 'jruby-${{ matrix.jruby_version }}' bundler-cache: true - name: show ruby version run: ruby -v From ed1e041324e16c5af91c0dad8b059b41fb97d2f2 Mon Sep 17 00:00:00 2001 From: joker1007 Date: Sun, 3 Dec 2023 19:28:26 +0900 Subject: [PATCH 14/28] ci: fix embulk gem version installed by bundler to fix CI error --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 824ce0e..21b916c 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source 'https://rubygems.org/' gemspec -gem 'embulk', '> 0.10.0' +gem 'embulk', '= 0.10.49' gem 'embulk-parser-none' gem 'embulk-parser-jsonl' gem 'pry-nav' From 246a087e3d3ff78a69bca5f8308735ae05dd02ca Mon Sep 17 00:00:00 2001 From: joker1007 Date: Sun, 3 Dec 2023 19:39:22 +0900 Subject: [PATCH 15/28] chore: delete travis CI badge --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 5818b61..b1df12a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # embulk-output-bigquery -[![Build Status](https://secure.travis-ci.org/embulk/embulk-output-bigquery.png?branch=master)](http://travis-ci.org/embulk/embulk-output-bigquery) - [Embulk](https://github.com/embulk/embulk/) output plugin to load/insert data into [Google BigQuery](https://cloud.google.com/bigquery/) using [direct insert](https://cloud.google.com/bigquery/loading-data-into-bigquery#loaddatapostrequest) ## Overview From 22869986bd3f5c60b1646f27ab83ae61d97b52e2 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Mon, 4 Dec 2023 22:05:44 +0900 Subject: [PATCH 16/28] Fix CI error with JRuby 9.4 test. Avoids the following error. You have already activated rake 13.0.6, but your Gemfile requires rake 13.1.0. --- .github/workflows/check.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 9b85ab9..68f1dcf 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -27,6 +27,15 @@ jobs: run: ruby -v - name: bundle install run: bundle install + # + # This step avoids the following error in the JRuby 9.4 test. + # + # Gem::LoadError: You have already activated rake 13.0.6, + # but your Gemfile requires rake 13.1.0. Prepending + # `bundle exec` to your command may solve this. + # + - name: install rake 13.1.0 + run: gem install rake -v 13.1.0 - name: install embulk.jar run: "curl -L -o embulk.jar https://github.com/embulk/embulk/releases/download/v0.10.49/embulk-0.10.49.jar" - name: rake test From ea9bce57c1b6f0f668959e57e5000371639a3995 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Wed, 3 Jan 2024 22:42:02 +0900 Subject: [PATCH 17/28] Add GitHub Actions for publish to Rubygems --- .github/workflows/publish.yml | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..e7f8781 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,40 @@ +name: Publish +on: + push: + tags: + - "v0.*" +jobs: + publish: + runs-on: ubuntu-latest + strategy: + fail-fast: true + steps: + - uses: actions/checkout@v4 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: 3.3.0 + # get tag variable using {{ github.ref_name }} + # + # References: + # * https://docs.github.com/en/actions/learn-github-actions/contexts#github-context + # * https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables + - name: extract gem version from tag + id: vars + run: echo version=${{ github.ref_name }} | sed -e 's/v0/0/' >> $GITHUB_OUTPUT + # + # From gem push documents. + # + # The push command will use ~/.gem/credentials to authenticate to a server, + # but you can use the RubyGems environment variable GEM_HOST_API_KEY + # to set the api key to authenticate. + # + # https://guides.rubygems.org/command-reference/#gem-push + # + - name: Publish + run: | + rake build + gem push pkg/${EMBULK_PLUGIN_NAME}-${{ steps.vars.outputs.version }}.gem + env: + EMBULK_PLUGIN_NAME: embulk-output-bigquery + GEM_HOST_API_KEY: "${{secrets.GEM_HOST_API_KEY}}" From 828e8ab932984c2e43f300fecb4dad8ea453811b Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Tue, 9 Jan 2024 22:03:07 +0900 Subject: [PATCH 18/28] Rename the secret name of RubyGems API to keep the consistency of other plugins --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e7f8781..bee5d57 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -37,4 +37,4 @@ jobs: gem push pkg/${EMBULK_PLUGIN_NAME}-${{ steps.vars.outputs.version }}.gem env: EMBULK_PLUGIN_NAME: embulk-output-bigquery - GEM_HOST_API_KEY: "${{secrets.GEM_HOST_API_KEY}}" + GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_API_KEY}}" From 12628179df9175c6f2119d926492fd908d798047 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Thu, 1 Feb 2024 19:27:38 +0900 Subject: [PATCH 19/28] Bump up 0.7.0 --- CHANGELOG.md | 3 +++ README.md | 7 +++++++ embulk-output-bigquery.gemspec | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f879f7e..c9df9fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 0.7.0 - 2024-02-1 +* [enhancement] Add support Embulk 0.11.x + ## 0.6.9 - 2023-03-16 * [enhancement] Add SSLException to retry job (thanks to @mzumi) diff --git a/README.md b/README.md index b1df12a..02ea9cf 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,13 @@ https://developers.google.com/bigquery/loading-data-into-bigquery * **Cleanup supported**: no * **Dynamic table creating**: yes +### Supported Embulk + +| gem version | Embulk version | +|------------------|--------------------| +| 0.7.0 and higher | v0.11.0 and higher | +| 0.6.9 and lower | v0.9.X and lower | + ### NOT IMPLEMENTED * insert data over streaming inserts * for continuous real-time insertions diff --git a/embulk-output-bigquery.gemspec b/embulk-output-bigquery.gemspec index df13d27..4f44258 100644 --- a/embulk-output-bigquery.gemspec +++ b/embulk-output-bigquery.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |spec| spec.name = "embulk-output-bigquery" - spec.version = "0.6.9" + spec.version = "0.7.0" spec.authors = ["Satoshi Akama", "Naotoshi Seo"] spec.summary = "Google BigQuery output plugin for Embulk" spec.description = "Embulk plugin that insert records to Google BigQuery." From e85e0a42824f8bf0d9f4e49bc9c68ae1f29edbcd Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Mon, 12 Feb 2024 23:51:15 +0900 Subject: [PATCH 20/28] Add missing GitHub Actions environment setting. --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bee5d57..a9a94e2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,6 +6,7 @@ on: jobs: publish: runs-on: ubuntu-latest + environment: maven-central-and-ruby-gems strategy: fail-fast: true steps: From 84d44f73133787b251db165aefdc5c985d399800 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Thu, 29 Feb 2024 20:53:33 +0900 Subject: [PATCH 21/28] Update GitHub Actions use checkout@v4 and setup-java@v4 --- .github/workflows/check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 68f1dcf..dea6b63 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -13,9 +13,9 @@ jobs: - 9.4.2.0 fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up OpenJDK 8 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: 8 distribution: "temurin" From 602103c80f153a46cdad430731ee5ee8649c5365 Mon Sep 17 00:00:00 2001 From: "nozomu.tsuruta" Date: Fri, 1 Mar 2024 18:47:39 +0900 Subject: [PATCH 22/28] Replace google-api-client with specific Google APIs --- embulk-output-bigquery.gemspec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/embulk-output-bigquery.gemspec b/embulk-output-bigquery.gemspec index 4f44258..b94df3b 100644 --- a/embulk-output-bigquery.gemspec +++ b/embulk-output-bigquery.gemspec @@ -15,7 +15,8 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] # the latest version - spec.add_dependency 'google-api-client','= 0.53.0' + spec.add_dependency 'google-apis-storage_v1' + spec.add_dependency 'google-apis-bigquery_v2' spec.add_dependency 'time_with_zone' spec.add_dependency 'thwait' # activesupport require Ruby >= 2.7.0 From dd311f54a0a977448345e8f15f22f7040f1a900c Mon Sep 17 00:00:00 2001 From: fagai Date: Sat, 2 Mar 2024 00:31:23 +0900 Subject: [PATCH 23/28] add config 'description' --- lib/embulk/output/bigquery.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/embulk/output/bigquery.rb b/lib/embulk/output/bigquery.rb index dd9b92d..10cbd00 100644 --- a/lib/embulk/output/bigquery.rb +++ b/lib/embulk/output/bigquery.rb @@ -63,6 +63,8 @@ def self.configure(config, schema, task_count) 'payload_column' => config.param('payload_column', :string, :default => nil), 'payload_column_index' => config.param('payload_column_index', :integer, :default => nil), + 'description' => config.param('description', :string, :default => nil), + 'open_timeout_sec' => config.param('open_timeout_sec', :integer, :default => nil), 'timeout_sec' => config.param('timeout_sec', :integer, :default => nil), # google-api-ruby-client < v0.11.0 'send_timeout_sec' => config.param('send_timeout_sec', :integer, :default => nil), # google-api-ruby-client >= v0.11.0 From 254ae192641e9b39cfae2236d2e2d60b091b0a66 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Mon, 4 Mar 2024 17:02:33 +0900 Subject: [PATCH 24/28] Bump up 0.7.1 --- CHANGELOG.md | 6 ++++++ embulk-output-bigquery.gemspec | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9df9fb..c92fa2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.7.1 - 2024-03-4 +* [enhancement] Support description of columns and tables (Thanks to @kyoshidajp and @fagai ) #142 +* [maintenance] Add missing GitHub Actions environment setting. #160 +* [maintenance] Replace google-api-client with specific Google APIs (Thanks to @Nozomuts) #161 +* [maintenance] Update GitHub Actions use checkout@v4 and setup-java@v4 #162 + ## 0.7.0 - 2024-02-1 * [enhancement] Add support Embulk 0.11.x diff --git a/embulk-output-bigquery.gemspec b/embulk-output-bigquery.gemspec index b94df3b..9606a7b 100644 --- a/embulk-output-bigquery.gemspec +++ b/embulk-output-bigquery.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |spec| spec.name = "embulk-output-bigquery" - spec.version = "0.7.0" + spec.version = "0.7.1" spec.authors = ["Satoshi Akama", "Naotoshi Seo"] spec.summary = "Google BigQuery output plugin for Embulk" spec.description = "Embulk plugin that insert records to Google BigQuery." From 1bad5d88f33bb7c9a5a98bf3b6c888cf2158b18a Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Fri, 12 Jul 2024 21:44:43 +0900 Subject: [PATCH 25/28] Fix GitHub Actions * Fix GitHub Actions Error You have already activated rake 13.1.0, but your Gemfile requires rake 13.2.1. * Update JRuby to 9.4.8.0 and 9.3.15.0. * Update Embulk to 0.11.4. --- .github/workflows/check.yml | 29 ++++++++++------------------- Gemfile | 2 +- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index dea6b63..b78bacd 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -9,8 +9,8 @@ jobs: strategy: matrix: jruby_version: - - 9.3.10.0 - - 9.4.2.0 + - 9.3.15.0 + - 9.4.8.0 fail-fast: false steps: - uses: actions/checkout@v4 @@ -19,24 +19,15 @@ jobs: with: java-version: 8 distribution: "temurin" - - uses: ruby/setup-ruby@v1 - with: - ruby-version: 'jruby-${{ matrix.jruby_version }}' - bundler-cache: true - - name: show ruby version - run: ruby -v - - name: bundle install - run: bundle install + - name: download jruby + run: "curl -L -o jruby.jar https://repo1.maven.org/maven2/org/jruby/jruby-complete/${{ matrix.jruby_version }}/jruby-complete-${{ matrix.jruby_version }}.jar" # - # This step avoids the following error in the JRuby 9.4 test. + # For avoiding permission denied. install gems into `gems` directory # - # Gem::LoadError: You have already activated rake 13.0.6, - # but your Gemfile requires rake 13.1.0. Prepending - # `bundle exec` to your command may solve this. - # - - name: install rake 13.1.0 - run: gem install rake -v 13.1.0 + - name: bundle install + run: "env GEM_HOME=gems java -jar jruby.jar -S bundle install" + - name: install embulk.jar - run: "curl -L -o embulk.jar https://github.com/embulk/embulk/releases/download/v0.10.49/embulk-0.10.49.jar" + run: "curl -L -o embulk.jar https://github.com/embulk/embulk/releases/download/v0.11.4/embulk-0.11.4.jar" - name: rake test - run: bundle exec env RUBYOPT="-r ./embulk.jar" rake test + run: 'env GEM_HOME=gems RUBYOPT="-r ./embulk.jar -r rubygems" java -jar jruby.jar -S bundle exec rake test' diff --git a/Gemfile b/Gemfile index 21b916c..f3dd9e3 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source 'https://rubygems.org/' gemspec -gem 'embulk', '= 0.10.49' +gem 'embulk', '= 0.11.4' gem 'embulk-parser-none' gem 'embulk-parser-jsonl' gem 'pry-nav' From 5ed9f0d6879a43d262a614c74091cbc140871b65 Mon Sep 17 00:00:00 2001 From: kashira Date: Fri, 16 Dec 2022 16:35:03 +0900 Subject: [PATCH 26/28] if table does not exist, embulk creates table on destination project --- lib/embulk/output/bigquery/bigquery_client.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/embulk/output/bigquery/bigquery_client.rb b/lib/embulk/output/bigquery/bigquery_client.rb index 2fcdb5e..66400a0 100644 --- a/lib/embulk/output/bigquery/bigquery_client.rb +++ b/lib/embulk/output/bigquery/bigquery_client.rb @@ -447,8 +447,8 @@ def create_table_if_not_exists(table, dataset: nil, options: nil) end opts = {} - Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" } - with_network_retry { client.insert_table(@project, dataset, body, **opts) } + Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@destination_project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" } + with_network_retry { client.insert_table(@destination_project, dataset, body, **opts) } rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e if e.status_code == 409 && /Already Exists:/ =~ e.message # ignore 'Already Exists' error @@ -457,7 +457,7 @@ def create_table_if_not_exists(table, dataset: nil, options: nil) response = {status_code: e.status_code, message: e.message, error_class: e.class} Embulk.logger.error { - "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts}), response:#{response}" + "embulk-output-bigquery: insert_table(#{@destination_project}, #{dataset}, #{@location_for_log}, #{body}, #{opts}), response:#{response}" } raise Error, "failed to create table #{@destination_project}:#{dataset}.#{table} in #{@location_for_log}, response:#{response}" end From 2c1c9f601a48a8a731afdfbfcac56bf2bf6b2dad Mon Sep 17 00:00:00 2001 From: kashira Date: Wed, 10 Jul 2024 23:08:25 +0900 Subject: [PATCH 27/28] use double splat on keyword arguments --- lib/embulk/output/bigquery/gcs_client.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/embulk/output/bigquery/gcs_client.rb b/lib/embulk/output/bigquery/gcs_client.rb index 0e0b57f..c2bf162 100644 --- a/lib/embulk/output/bigquery/gcs_client.rb +++ b/lib/embulk/output/bigquery/gcs_client.rb @@ -48,7 +48,7 @@ def insert_temporary_bucket(bucket = nil) opts = {} Embulk.logger.debug { "embulk-output-bigquery: insert_temporary_bucket(#{@project}, #{body}, #{opts})" } - with_network_retry { client.insert_bucket(@project, body, opts) } + with_network_retry { client.insert_bucket(@project, body, **opts) } rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e if e.status_code == 409 && /conflict:/ =~ e.message # ignore 'Already Exists' error @@ -81,7 +81,7 @@ def insert_object(path, object: nil, bucket: nil) Embulk.logger.debug { "embulk-output-bigquery: insert_object(#{bucket}, #{body}, #{opts})" } # memo: gcs is strongly consistent for insert (read-after-write). ref: https://cloud.google.com/storage/docs/consistency - with_network_retry { client.insert_object(bucket, body, opts) } + with_network_retry { client.insert_object(bucket, body, **opts) } rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e response = {status_code: e.status_code, message: e.message, error_class: e.class} Embulk.logger.error { @@ -114,7 +114,7 @@ def delete_object(object, bucket: nil) opts = {} Embulk.logger.debug { "embulk-output-bigquery: delete_object(#{bucket}, #{object}, #{opts})" } - response = with_network_retry { client.delete_object(bucket, object, opts) } + response = with_network_retry { client.delete_object(bucket, object, **opts) } rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e if e.status_code == 404 # ignore 'notFound' error return nil From 738ba1b449a2ad7a37e2eac60a5b8dad715ea970 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sun, 21 Jul 2024 11:45:02 +0900 Subject: [PATCH 28/28] Bump up 0.7.2 --- CHANGELOG.md | 5 +++++ embulk-output-bigquery.gemspec | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c92fa2c..e589949 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.7.2 - 2024-07-21 +* [maintenance] Fix GitHub Actions #166 +* [maintenance] Fix gcs_client in order to load data using gcs_bucket parameter (Thanks to kashira202111) #164 +* [maintenance] Prevent creating unnecessary tables. (Thanks to kashira202111) #148 + ## 0.7.1 - 2024-03-4 * [enhancement] Support description of columns and tables (Thanks to @kyoshidajp and @fagai ) #142 * [maintenance] Add missing GitHub Actions environment setting. #160 diff --git a/embulk-output-bigquery.gemspec b/embulk-output-bigquery.gemspec index 9606a7b..fa7c7a1 100644 --- a/embulk-output-bigquery.gemspec +++ b/embulk-output-bigquery.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |spec| spec.name = "embulk-output-bigquery" - spec.version = "0.7.1" + spec.version = "0.7.2" spec.authors = ["Satoshi Akama", "Naotoshi Seo"] spec.summary = "Google BigQuery output plugin for Embulk" spec.description = "Embulk plugin that insert records to Google BigQuery."