Added merge mode

[t3t5u]

create table

CREATE TABLE `test_by_sano`.`output` (
  `key_long` INT64 NOT NULL,
  `key_string` STRING NOT NULL,
  `test_boolean` BOOL,
  `test_long` INT64,
  `test_double` FLOAT64,
  `test_string` STRING,
  `test_timestamp` TIMESTAMP,
  `test_json` STRING,
  PRIMARY KEY(`key_long`, `key_string`) NOT ENFORCED
);

test jsonl

$ cat test_1.jsonl
{"key_long":1,"key_string":"a","test_boolean":true,"test_long":123,"test_double":1.23,"test_string":"あいうえお","test_timestamp":"1999-12-31 23:59:59.000000 +0000","test_json":{"キー":"値"}}
{"key_long":2,"key_string":"b","test_boolean":false,"test_long":456,"test_double":4.56,"test_string":"かきくけこ","test_timestamp":"2000-01-01 00:00:00.000000 +0000","test_json":[{"キー1":"値1"},{"キー2":"値2"}]}
{"key_long":3,"key_string":"c","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":null}
{"key_long":4,"key_string":"d","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":12345}
{"key_long":5,"key_string":"e","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":123.45}
{"key_long":6,"key_string":"f","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":"hoge"}

$ cat test_2.jsonl
{"key_long":1,"key_string":"a","test_boolean":true,"test_long":123,"test_double":1.23,"test_string":"あいうえお","test_timestamp":"1999-12-31 23:59:59.000000 +0000","test_json":{"キー":"値"}}
{"key_long":3,"key_string":"c","test_boolean":false,"test_long":456,"test_double":4.56,"test_string":"かきくけこ","test_timestamp":"2000-01-01 00:00:00.000000 +0000","test_json":[{"キー1":"値1"},{"キー2":"値2"}]}
{"key_long":5,"key_string":"e","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":null}
{"key_long":7,"key_string":"g","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":12345}
{"key_long":8,"key_string":"h","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":123.45}
{"key_long":9,"key_string":"i","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":"hoge"}

config

$ cat config_out.yml
in:
  type: file
  path_prefix: test_1.jsonl
  parser:
    type: jsonl
    columns:
    - {name: key_long, type: long}
    - {name: key_string, type: string}
    - {name: test_boolean, type: boolean}
    - {name: test_long, type: long}
    - {name: test_double, type: double}
    - {name: test_string, type: string}
    - {name: test_timestamp, type: timestamp}
    - {name: test_json, type: json}
out:
  type: bigquery
  auth_method: service_account
  json_keyfile: key.json
  dataset: test_by_sano
  table: output
  location: US
  compression: GZIP
  source_format: NEWLINE_DELIMITED_JSON
  mode: merge
  merge_keys: []
  merge_rule: []

$ cat config_in.yml
in:
  type: bigquery
  project: systemn-playground
  keyfile: key.json
  sql: SELECT * FROM `test_by_sano`.`output` ORDER BY `key_long`, `key_string`
  location: US
out:
  type: command
  command: 'cat -'
  formatter:
    type: jsonl

run test 1

$ embulk run config_out.yml
2024-07-19 11:00:27.259 +0000: Embulk v0.9.26
2024-07-19 11:00:27.892 +0000 [WARN] (main): DEPRECATION: JRuby org.jruby.embed.ScriptingContainer is directly injected.
2024-07-19 11:00:29.189 +0000 [INFO] (main): Gem's home and path are set by default: "/home/ubuntu/.embulk/lib/gems"
2024-07-19 11:00:29.797 +0000 [INFO] (main): Started Embulk v0.9.26
2024-07-19 11:00:32.374 +0000 [INFO] (0001:transaction): Loaded plugin embulk-output-bigquery (0.7.1)
2024-07-19 11:00:32.409 +0000 [INFO] (0001:transaction): Loaded plugin embulk-parser-jsonl (0.2.1)
2024-07-19 11:00:32.421 +0000 [INFO] (0001:transaction): Listing local files at directory '.' filtering filename by prefix 'test_1.jsonl'
2024-07-19 11:00:32.421 +0000 [INFO] (0001:transaction): "follow_symlinks" is set false. Note that symbolic links to directories are skipped.
2024-07-19 11:00:32.425 +0000 [INFO] (0001:transaction): Loading files [test_1.jsonl]
2024-07-19 11:00:32.444 +0000 [INFO] (0001:transaction): Using local thread executor with max_threads=16 / output tasks 8 = input tasks 1 * 8
2024-07-19 11:00:32.477 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Get dataset... systemn-playground:test_by_sano
2024-07-19 11:00:33.718 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Create table... systemn-playground:test_by_sano.LOAD_TEMP_a1848ee6_034f_4929_9f57_66ef4226c465_output
2024-07-19 11:00:34.140 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Create table... systemn-playground:test_by_sano.output
2024-07-19 11:00:34.488 +0000 [INFO] (0001:transaction): {done:  0 / 1, running: 0}
2024-07-19 11:00:34.592 +0000 [INFO] (embulk-output-executor-0): embulk-output-bigquery: create /tmp/embulk_output_bigquery_20240719-17924-gnor1d.17924.2004.jsonl.gz
2024-07-19 11:00:34.631 +0000 [INFO] (0001:transaction): {done:  1 / 1, running: 0}
2024-07-19 11:00:34.643 +0000 [INFO] (Ruby-0-Thread-1: /home/ubuntu/.embulk/lib/gems/gems/embulk-output-bigquery-0.7.1/lib/embulk/output/bigquery/bigquery_client.rb:153): embulk-output-bigquery: Load job starting... job_id:[embulk_load_job_6c03d504-0710-4561-bf5e-e6351ef4f7c7] /tmp/embulk_output_bigquery_20240719-17924-gnor1d.17924.2004.jsonl.gz => systemn-playground:test_by_sano.LOAD_TEMP_a1848ee6_034f_4929_9f57_66ef4226c465_output in US
2024-07-19 11:00:37.462 +0000 [INFO] (Ruby-0-Thread-1: /home/ubuntu/.embulk/lib/gems/gems/embulk-output-bigquery-0.7.1/lib/embulk/output/bigquery/bigquery_client.rb:153): embulk-output-bigquery: Load job checking... job_id:[embulk_load_job_6c03d504-0710-4561-bf5e-e6351ef4f7c7] elapsed_time:6.2e-05sec status:[RUNNING]
2024-07-19 11:00:47.748 +0000 [INFO] (Ruby-0-Thread-1: /home/ubuntu/.embulk/lib/gems/gems/embulk-output-bigquery-0.7.1/lib/embulk/output/bigquery/bigquery_client.rb:153): embulk-output-bigquery: Load job completed... job_id:[embulk_load_job_6c03d504-0710-4561-bf5e-e6351ef4f7c7] elapsed_time:10.288134sec status:[DONE]
2024-07-19 11:00:47.749 +0000 [INFO] (Ruby-0-Thread-1: /home/ubuntu/.embulk/lib/gems/gems/embulk-output-bigquery-0.7.1/lib/embulk/output/bigquery/bigquery_client.rb:153): embulk-output-bigquery: Load job response... job_id:[embulk_load_job_6c03d504-0710-4561-bf5e-e6351ef4f7c7] response.statistics:{:start_time=>1721386837263, :completion_ratio=>1, :creation_time=>1721386837046, :end_time=>1721386838761, :reservation_id=>"default-pipeline", :total_slot_ms=>84, :load=>{:input_file_bytes=>1039, :output_bytes=>232, :output_rows=>6, :bad_records=>0, :input_files=>1}}
2024-07-19 11:00:47.752 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Get table... systemn-playground:test_by_sano.LOAD_TEMP_a1848ee6_034f_4929_9f57_66ef4226c465_output
2024-07-19 11:00:48.119 +0000 [INFO] (0001:transaction): embulk-output-bigquery: transaction_report: {"num_input_rows":6,"num_response_rows":6,"num_output_rows":6,"num_rejected_rows":0}
2024-07-19 11:00:48.121 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job starting... job_id:[embulk_query_job_9d048976-7bea-4461-90ef-2bc585fec995]
2024-07-19 11:00:49.762 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job checking... job_id:[embulk_query_job_9d048976-7bea-4461-90ef-2bc585fec995] elapsed_time:5.9e-05sec status:[RUNNING]
2024-07-19 11:01:00.088 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job completed... job_id:[embulk_query_job_9d048976-7bea-4461-90ef-2bc585fec995] elapsed_time:10.325949000000001sec status:[DONE]
2024-07-19 11:01:00.098 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job response... job_id:[embulk_query_job_9d048976-7bea-4461-90ef-2bc585fec995] response.statistics:{:start_time=>1721386848410, :creation_time=>1721386848289, :end_time=>1721386849880, :total_slot_ms=>40329, :total_bytes_processed=>20971520, :query=>{:billing_tier=>1, :referenced_tables=>[{:project_id=>"systemn-playground", :dataset_id=>"test_by_sano", :table_id=>"INFORMATION_SCHEMA.KEY_COLUMN_USAGE"}, {:project_id=>"systemn-playground", :dataset_id=>"test_by_sano", :table_id=>"INFORMATION_SCHEMA.KEY_COLUMN_USAGE"}, {:project_id=>"systemn-playground", :dataset_id=>"test_by_sano", :table_id=>"INFORMATION_SCHEMA.TABLE_CONSTRAINTS"}, {:project_id=>"systemn-playground", :dataset_id=>"test_by_sano", :table_id=>"INFORMATION_SCHEMA.TABLE_CONSTRAINTS"}], :query_plan=>[{:wait_ms_max=>341, :wait_ratio_max=>1, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>4, :wait_ms_avg=>185, :write_ratio_max=>0.03225806451612903, :completed_parallel_inputs=>15, :records_written=>1, :id=>0, :compute_ratio_avg=>0.05278592375366569, :read_ms_max=>0, :parallel_inputs=>15, :compute_ratio_max=>0.08211143695014662, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$10:pk_uuid", "FROM systemn-playground.region-US.INFORMATION_SCHEMA.TABLE_CONSTRAINTS_REFERENCED_DATASET_CONSTRAINTS_TABLE"]}, {:kind=>"WRITE", :substeps=>["$10", "TO __stage00_output"]}], :status=>"COMPLETE", :compute_ms_max=>28, :end_ms=>1721386848991, :name=>"S00: Input", :start_ms=>1721386848547, :write_ratio_avg=>0.011730205278592375, :write_ms_max=>11, :compute_ms_avg=>18, :records_read=>1, :shuffle_output_bytes=>41, :read_ratio_avg=>0, :wait_ratio_avg=>0.5425219941348973}, {:wait_ms_max=>332, :wait_ratio_max=>0.9736070381231672, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>6, :wait_ms_avg=>207, :write_ratio_max=>0.06744868035190615, :completed_parallel_inputs=>15, :records_written=>1, :id=>1, :compute_ratio_avg=>0.04105571847507331, :read_ms_max=>0, :parallel_inputs=>15, :compute_ratio_max=>0.06158357771260997, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$1:pk_uuid", "FROM systemn-playground.region-US.INFORMATION_SCHEMA.TABLE_CONSTRAINTS_REFERENCED_DATASET_CONSTRAINTS_TABLE"]}, {:kind=>"WRITE", :substeps=>["$1", "TO __stage01_output"]}], :status=>"COMPLETE", :compute_ms_max=>21, :end_ms=>1721386848970, :name=>"S01: Input", :start_ms=>1721386848572, :write_ratio_avg=>0.017595307917888565, :write_ms_max=>23, :compute_ms_avg=>14, :records_read=>1, :shuffle_output_bytes=>41, :read_ratio_avg=>0, :wait_ratio_avg=>0.6070381231671554}, {:wait_ms_max=>188, :wait_ratio_max=>0.5513196480938416, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>2, :wait_ms_avg=>141, :write_ratio_max=>0.1466275659824047, :completed_parallel_inputs=>100, :records_written=>1, :id=>2, :compute_ratio_avg=>0.020527859237536656, :read_ms_max=>0, :input_stages=>["1"], :parallel_inputs=>100, :compute_ratio_max=>0.04398826979472141, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["FROM __stage01_output"]}], :status=>"COMPLETE", :compute_ms_max=>15, :end_ms=>1721386849305, :name=>"S02: Coalesce", :start_ms=>1721386848972, :write_ratio_avg=>0.005865102639296188, :write_ms_max=>50, :compute_ms_avg=>7, :records_read=>1, :shuffle_output_bytes=>41, :read_ratio_avg=>0, :wait_ratio_avg=>0.41348973607038125}, {:wait_ms_max=>175, :wait_ratio_max=>0.5131964809384164, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>4, :wait_ms_avg=>175, :write_ratio_max=>0.011730205278592375, :completed_parallel_inputs=>1, :records_written=>2, :id=>3, :compute_ratio_avg=>0.0469208211143695, :read_ms_max=>0, :input_stages=>["2"], :parallel_inputs=>1, :compute_ratio_max=>0.0469208211143695, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$20:constraint_catalog, $21:constraint_schema, $22:constraint_name, $23:table_catalog, $24:table_schema, $25:table_name, $26:column_name, $27:ordinal_position, $28:pk_uuid", "FROM systemn-playground.test_by_sano.INFORMATION_SCHEMA.KEY_COLUMN_USAGE_DATASET_KEY_COLUMN_USAGE_TABLE", "WHERE equal($25, 'output')"]}, {:kind=>"READ", :substeps=>["$1", "FROM __stage02_output"]}, {:kind=>"JOIN", :substeps=>["$50 := $20, $51 := $21, $52 := $22, $53 := $23, $54 := $24, $55 := $25, $56 := $26, $57 := $27", "SEMI HASH JOIN EACH  WITH ALL  ON $28 = $1"]}, {:kind=>"WRITE", :substeps=>["$50, $51, $52, $53, $54, $55, $56, $57", "TO __stage03_output"]}], :status=>"COMPLETE", :compute_ms_max=>16, :end_ms=>1721386849318, :name=>"S03: Join", :start_ms=>1721386849168, :write_ratio_avg=>0.011730205278592375, :write_ms_max=>4, :compute_ms_avg=>16, :records_read=>3, :shuffle_output_bytes=>258, :read_ratio_avg=>0, :wait_ratio_avg=>0.5131964809384164}, {:wait_ms_max=>205, :wait_ratio_max=>0.6011730205278593, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>3, :wait_ms_avg=>153, :write_ratio_max=>0.1495601173020528, :completed_parallel_inputs=>100, :records_written=>1, :id=>4, :compute_ratio_avg=>0.017595307917888565, :read_ms_max=>0, :input_stages=>["0"], :parallel_inputs=>100, :compute_ratio_max=>0.03812316715542522, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["FROM __stage00_output"]}], :status=>"COMPLETE", :compute_ms_max=>13, :end_ms=>1721386849295, :name=>"S04: Coalesce", :start_ms=>1721386849000, :write_ratio_avg=>0.008797653958944282, :write_ms_max=>51, :compute_ms_avg=>6, :records_read=>1, :shuffle_output_bytes=>41, :read_ratio_avg=>0, :wait_ratio_avg=>0.44868035190615835}, {:wait_ms_max=>61, :wait_ratio_max=>0.17888563049853373, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>3, :wait_ms_avg=>38, :write_ratio_max=>0.15542521994134897, :completed_parallel_inputs=>100, :records_written=>2, :id=>6, :compute_ratio_avg=>0.02346041055718475, :read_ms_max=>0, :input_stages=>["3"], :parallel_inputs=>100, :compute_ratio_max=>0.04105571847507331, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["FROM __stage03_output"]}], :status=>"COMPLETE", :compute_ms_max=>14, :end_ms=>1721386849506, :name=>"S06: Coalesce", :start_ms=>1721386849325, :write_ratio_avg=>0.008797653958944282, :write_ms_max=>53, :compute_ms_avg=>8, :records_read=>2, :shuffle_output_bytes=>258, :read_ratio_avg=>0, :wait_ratio_avg=>0.11143695014662756}, {:wait_ms_max=>63, :wait_ratio_max=>0.18475073313782991, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>5, :wait_ms_avg=>63, :write_ratio_max=>0.01466275659824047, :completed_parallel_inputs=>1, :records_written=>2, :id=>7, :compute_ratio_avg=>0.05865102639296188, :read_ms_max=>0, :input_stages=>["4", "6"], :parallel_inputs=>1, :compute_ratio_max=>0.05865102639296188, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$30:constraint_catalog, $31:constraint_schema, $32:constraint_name, $33:table_catalog, $34:table_schema, $35:table_name, $37:pk_uuid, $36:constraint_type", "FROM systemn-playground.test_by_sano.INFORMATION_SCHEMA.TABLE_CONSTRAINTS_REFERENCING_DATASET_CONSTRAINTS_TABLE", "WHERE and(equal($35, 'output'), equal($36, 'PRIMARY KEY'))"]}, {:kind=>"READ", :substeps=>["$10", "FROM __stage04_output"]}, {:kind=>"READ", :substeps=>["$50, $51, $52, $53, $54, $55, $56, $57", "FROM __stage06_output"]}, {:kind=>"JOIN", :substeps=>["$80 := $56, $81 := $57", "INNER HASH JOIN EACH  WITH ALL  ON $70 = $50, $71 = $51, $72 = $52, $73 = $53, $74 = $54, $75 = $55"]}, {:kind=>"JOIN", :substeps=>["$70 := $30, $71 := $31, $72 := $32, $73 := $33, $74 := $34, $75 := $35", "SEMI HASH JOIN EACH  WITH ALL  ON $37 = $10"]}, {:kind=>"WRITE", :substeps=>["$80, $81", "TO __stage07_output"]}], :status=>"COMPLETE", :compute_ms_max=>20, :end_ms=>1721386849645, :name=>"S07: Join", :start_ms=>1721386849382, :write_ratio_avg=>0.01466275659824047, :write_ms_max=>5, :compute_ms_avg=>20, :records_read=>4, :shuffle_output_bytes=>46, :read_ratio_avg=>0, :wait_ratio_avg=>0.18475073313782991}, {:wait_ms_max=>2, :wait_ratio_max=>0.005865102639296188, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>1, :wait_ms_avg=>2, :write_ratio_max=>0.002932551319648094, :completed_parallel_inputs=>1, :records_written=>2, :id=>8, :compute_ratio_avg=>0.017595307917888565, :read_ms_max=>0, :input_stages=>["7"], :parallel_inputs=>1, :compute_ratio_max=>0.017595307917888565, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$80, $81", "FROM __stage07_output"]}, {:kind=>"SORT", :substeps=>["$81 ASC"]}, {:kind=>"WRITE", :substeps=>["$90", "TO __stage08_output"]}], :status=>"COMPLETE", :compute_ms_max=>6, :end_ms=>1721386849660, :name=>"S08: Output", :start_ms=>1721386849650, :write_ratio_avg=>0.002932551319648094, :write_ms_max=>1, :compute_ms_avg=>6, :records_read=>2, :shuffle_output_bytes=>28, :read_ratio_avg=>0, :wait_ratio_avg=>0.005865102639296188}], :total_bytes_billed=>20971520, :estimated_bytes_processed=>20971520, :total_slot_ms=>40329, :total_bytes_processed=>20971520, :total_partitions_processed=>0, :cache_hit=>false, :timeline=>[{:total_slot_ms=>3186, :pending_units=>202, :elapsed_ms=>629, :active_units=>24, :completed_units=>30}, {:total_slot_ms=>40329, :pending_units=>0, :elapsed_ms=>1255, :active_units=>1, :completed_units=>333}], :statement_type=>"SELECT"}}
2024-07-19 11:01:00.493 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Execute query... MERGE `test_by_sano`.`output` T USING `test_by_sano`.`LOAD_TEMP_a1848ee6_034f_4929_9f57_66ef4226c465_output` S ON T.`key_long` = S.`key_long` AND T.`key_string` = S.`key_string` WHEN MATCHED THEN UPDATE SET T.`key_long` = S.`key_long`, T.`key_string` = S.`key_string`, T.`test_boolean` = S.`test_boolean`, T.`test_long` = S.`test_long`, T.`test_double` = S.`test_double`, T.`test_string` = S.`test_string`, T.`test_timestamp` = S.`test_timestamp`, T.`test_json` = S.`test_json` WHEN NOT MATCHED THEN INSERT (`key_long`, `key_string`, `test_boolean`, `test_long`, `test_double`, `test_string`, `test_timestamp`, `test_json`) VALUES (`key_long`, `key_string`, `test_boolean`, `test_long`, `test_double`, `test_string`, `test_timestamp`, `test_json`)
2024-07-19 11:01:00.493 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job starting... job_id:[embulk_query_job_bb85f570-854a-4144-8586-6b8195c46298]
2024-07-19 11:01:01.348 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job checking... job_id:[embulk_query_job_bb85f570-854a-4144-8586-6b8195c46298] elapsed_time:5.8e-05sec status:[RUNNING]
2024-07-19 11:01:11.644 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job completed... job_id:[embulk_query_job_bb85f570-854a-4144-8586-6b8195c46298] elapsed_time:10.295684sec status:[DONE]
2024-07-19 11:01:11.671 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job response... job_id:[embulk_query_job_bb85f570-854a-4144-8586-6b8195c46298] response.statistics:{:start_time=>1721386861214, :creation_time=>1721386860704, :end_time=>1721386863564, :total_slot_ms=>27779, :total_bytes_processed=>232, :query=>{:billing_tier=>1, :referenced_tables=>[{:project_id=>"systemn-playground", :dataset_id=>"test_by_sano", :table_id=>"output"}, {:project_id=>"systemn-playground", :dataset_id=>"test_by_sano", :table_id=>"LOAD_TEMP_a1848ee6_034f_4929_9f57_66ef4226c465_output"}], :query_plan=>[{:wait_ms_max=>37, :wait_ratio_max=>0.19270833333333334, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>54, :wait_ms_avg=>37, :write_ratio_max=>0.28125, :completed_parallel_inputs=>1, :records_written=>0, :id=>0, :compute_ratio_avg=>0.026041666666666668, :read_ms_max=>0, :parallel_inputs=>1, :compute_ratio_max=>0.026041666666666668, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$10:key_long, $11:key_string, $12:$file_temp_id, $13:$row_temp_id", "FROM test_by_sano.output"]}, {:kind=>"COMPUTE", :substeps=>["$150 := 1"]}, {:kind=>"WRITE", :substeps=>["$10, $11, $12, $13, $150", "TO __stage00_output"]}], :status=>"COMPLETE", :compute_ms_max=>5, :end_ms=>1721386861675, :name=>"S00: Input", :start_ms=>1721386861435, :write_ratio_avg=>0.28125, :write_ms_max=>54, :compute_ms_avg=>5, :records_read=>0, :shuffle_output_bytes=>0, :read_ratio_avg=>0, :wait_ratio_avg=>0.19270833333333334}, {:wait_ms_max=>103, :wait_ratio_max=>0.5364583333333334, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>4, :wait_ms_avg=>72, :write_ratio_max=>0.2708333333333333, :completed_parallel_inputs=>100, :records_written=>0, :id=>1, :compute_ratio_avg=>0.046875, :read_ms_max=>0, :input_stages=>["0"], :parallel_inputs=>100, :compute_ratio_max=>0.078125, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["FROM __stage00_output"]}], :status=>"COMPLETE", :compute_ms_max=>15, :end_ms=>1721386861917, :name=>"S01: Coalesce", :start_ms=>1721386861716, :write_ratio_avg=>0.020833333333333332, :write_ms_max=>52, :compute_ms_avg=>9, :records_read=>0, :shuffle_output_bytes=>0, :read_ratio_avg=>0, :wait_ratio_avg=>0.375}, {:wait_ms_max=>118, :wait_ratio_max=>0.6145833333333334, :read_ms_avg=>3, :read_ratio_max=>0.015625, :write_ms_avg=>3, :wait_ms_avg=>118, :write_ratio_max=>0.015625, :completed_parallel_inputs=>1, :records_written=>6, :id=>2, :compute_ratio_avg=>0.057291666666666664, :read_ms_max=>3, :input_stages=>["1"], :parallel_inputs=>1, :compute_ratio_max=>0.057291666666666664, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$1:key_long, $2:key_string, $3:test_boolean, $4:test_long, $5:test_double, $6:test_string, $7:test_timestamp, $8:test_json", "FROM test_by_sano.LOAD_TEMP_a1848ee6_034f_4929_9f57_66ef4226c465_output"]}, {:kind=>"READ", :substeps=>["$10, $11, $12, $13, $150", "FROM __stage01_output"]}, {:kind=>"AGGREGATE", :substeps=>["GROUP BY $180 := $101, $181 := $102, $182 := $90", "$70 := ANY_AND_CHECK_FOR_UPDATE($121)", "$71 := ANY_AND_CHECK_FOR_UPDATE($120)", "$72 := ANY_AND_CHECK_FOR_UPDATE($122)", "$73 := ANY_AND_CHECK_FOR_UPDATE($123)", "$74 := ANY_AND_CHECK_FOR_UPDATE($124)", "$75 := ANY_AND_CHECK_FOR_UPDATE($125)", "$76 := ANY_AND_CHECK_FOR_UPDATE($126)", "$77 := ANY_AND_CHECK_FOR_UPDATE($127)", "$78 := ANY_AND_CHECK_FOR_UPDATE($128)", "$79 := ANY_AND_CHECK_FOR_UPDATE($129)", "$80 := ANY_AND_CHECK_FOR_UPDATE($141)", "$81 := ANY_AND_CHECK_FOR_UPDATE($142)", "$82 := ANY_AND_CHECK_FOR_UPDATE($143)", "$83 := ANY_AND_CHECK_FOR_UPDATE($144)", "$84 := ANY_AND_CHECK_FOR_UPDATE($145)", "$85 := ANY_AND_CHECK_FOR_UPDATE($146)"]}, {:kind=>"COMPUTE", :substeps=>["$90 := if(equal($120, 10001), $100, NULL)"]}, {:kind=>"COMPUTE", :substeps=>["$100 := UNIQUE_ROW_ID()", "$101 := if(equal($120, 10001), SAFE_EXPR(CAST(add(1000000000, abs(mod(farm_fingerprint($110), 1000000000))) AS UINT64)), $160)", "$102 := if(equal($120, 10001), 18446744073709551615, $161)"]}, {:kind=>"COMPUTE", :substeps=>["$110 := UNIQUE_ROW_ID()"]}, {:kind=>"FILTER", :substeps=>["not(equal($120, 0))"]}, {:kind=>"COMPUTE", :substeps=>["$120 := if($130, 10001, $140)", "$121 := if($130, 2, if(not(is_null($162)), 1, -1))", "$122 := if($130, $163, NULL)", "$123 := if($130, $164, NULL)", "$124 := if($130, $165, NULL)", "$125 := if($130, $166, NULL)", "$126 := if($130, $167, NULL)", "$127 := if($130, $168, NULL)", "$128 := if($130, $169, NULL)", "$129 := if($130, $170, NULL)"]}, {:kind=>"COMPUTE", :substeps=>["$130 := and(equal($140, 0), is_null($162))"]}, {:kind=>"COMPUTE", :substeps=>["$140 := if(not(is_null($162)), 10002, 0)", "$141 := if(not(is_null($162)), $163, NULL)", "$142 := if(not(is_null($162)), $164, NULL)", "$143 := if(not(is_null($162)), $165, NULL)", "$144 := if(not(is_null($162)), $166, NULL)", "$145 := if(not(is_null($162)), $167, NULL)", "$146 := if(not(is_null($162)), $168, NULL)", "$147 := if(not(is_null($162)), $169, NULL)", "$148 := if(not(is_null($162)), $170, NULL)"]}, {:kind=>"JOIN", :substeps=>["$163 := $1, $164 := $2, $165 := $3, $166 := $4, $167 := $5, $168 := $6, $169 := $7, $170 := $8, $160 := $12, $161 := $13, $162 := $150", "LEFT OUTER HASH JOIN EACH  WITH ALL  ON $1 = $10, $2 = $11"]}, {:kind=>"WRITE", :substeps=>["$70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85", "TO __stage02_output", "BY HASH($180, $181, $182)"]}], :status=>"COMPLETE", :compute_ms_max=>11, :end_ms=>1721386862031, :name=>"S02: Join+", :start_ms=>1721386861785, :write_ratio_avg=>0.015625, :write_ms_max=>3, :compute_ms_avg=>11, :records_read=>6, :shuffle_output_bytes=>756, :read_ratio_avg=>0.015625, :wait_ratio_avg=>0.6145833333333334}, {:wait_ms_max=>28, :wait_ratio_max=>0.14583333333333334, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>15, :wait_ms_avg=>28, :write_ratio_max=>0.078125, :completed_parallel_inputs=>1, :records_written=>6, :id=>3, :compute_ratio_avg=>0.03125, :read_ms_max=>0, :input_stages=>["2"], :parallel_inputs=>1, :compute_ratio_max=>0.03125, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85", "FROM __stage02_output"]}, {:kind=>"AGGREGATE", :substeps=>["GROUP BY $190 := $180, $191 := $181, $192 := $182", "$50 := ANY_AND_CHECK_FOR_UPDATE($70)", "$51 := ANY_AND_CHECK_FOR_UPDATE($71)", "$52 := ANY_AND_CHECK_FOR_UPDATE($72)", "$53 := ANY_AND_CHECK_FOR_UPDATE($73)", "$54 := ANY_AND_CHECK_FOR_UPDATE($74)", "$55 := ANY_AND_CHECK_FOR_UPDATE($75)", "$56 := ANY_AND_CHECK_FOR_UPDATE($76)", "$57 := ANY_AND_CHECK_FOR_UPDATE($77)", "$58 := ANY_AND_CHECK_FOR_UPDATE($78)", "$59 := ANY_AND_CHECK_FOR_UPDATE($79)", "$60 := ANY_AND_CHECK_FOR_UPDATE($80)", "$61 := ANY_AND_CHECK_FOR_UPDATE($81)", "$62 := ANY_AND_CHECK_FOR_UPDATE($82)", "$63 := ANY_AND_CHECK_FOR_UPDATE($83)", "$64 := ANY_AND_CHECK_FOR_UPDATE($84)", "$65 := ANY_AND_CHECK_FOR_UPDATE($85)"]}, {:kind=>"WRITE", :substeps=>["$50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, $64, $65", "TO __stage03_output", "BY HASH($190)"]}], :status=>"COMPLETE", :compute_ms_max=>6, :end_ms=>1721386862086, :name=>"S03: Aggregate", :start_ms=>1721386862063, :write_ratio_avg=>0.078125, :write_ms_max=>15, :compute_ms_avg=>6, :records_read=>6, :shuffle_output_bytes=>570, :read_ratio_avg=>0, :wait_ratio_avg=>0.14583333333333334}, {:wait_ms_max=>33, :wait_ratio_max=>0.171875, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>55, :wait_ms_avg=>33, :write_ratio_max=>0.2864583333333333, :completed_parallel_inputs=>1, :records_written=>6, :id=>4, :compute_ratio_avg=>0.0625, :read_ms_max=>0, :input_stages=>["3"], :parallel_inputs=>1, :compute_ratio_max=>0.0625, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, $64, $65", "FROM __stage03_output"]}, {:kind=>"COMPUTE", :substeps=>["$20 := if(equal(2, $30), $221, if(equal(1, $30), $229, $40))", "$21 := if(equal(2, $30), $222, if(equal(1, $30), $230, $41))", "$22 := if(equal(2, $30), $223, if(equal(1, $30), $231, $42))", "$23 := if(equal(2, $30), $224, if(equal(1, $30), $232, $43))", "$24 := if(equal(2, $30), $225, if(equal(1, $30), $233, $44))", "$25 := if(equal(2, $30), $226, if(equal(1, $30), $234, $45))", "$26 := if(equal(2, $30), $227, if(equal(1, $30), $235, $46))", "$27 := if(equal(2, $30), $228, if(equal(1, $30), $236, $47))", "$28 := if(equal(-1, $30), 'REINSERT', if(in($30, 1), 'UPDATE', NULL))"]}, {:kind=>"COMPUTE", :substeps=>["$30 := if(not($48), -1, $220)"]}, {:kind=>"SORT", :substeps=>["$190 ASC, $191 DESC"]}, {:kind=>"WRITE", :substeps=>["$20, $21, $22, $23, $24, $25, $26, $27, $28", "TO __stage04_output"]}], :status=>"COMPLETE", :compute_ms_max=>12, :end_ms=>1721386862302, :name=>"S04: Sort+", :start_ms=>1721386862132, :write_ratio_avg=>0.2864583333333333, :write_ms_max=>55, :compute_ms_avg=>12, :records_read=>6, :shuffle_output_bytes=>270, :read_ratio_avg=>0, :wait_ratio_avg=>0.171875}, {:wait_ms_max=>54, :wait_ratio_max=>0.28125, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>28, :wait_ms_avg=>53, :write_ratio_max=>0.2916666666666667, :completed_parallel_inputs=>50, :records_written=>6, :id=>5, :compute_ratio_avg=>0.052083333333333336, :read_ms_max=>0, :input_stages=>["4"], :parallel_inputs=>50, :compute_ratio_max=>0.08333333333333333, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["FROM __stage04_output"]}], :status=>"COMPLETE", :compute_ms_max=>16, :end_ms=>1721386862527, :name=>"S05: Coalesce", :start_ms=>1721386862360, :write_ratio_avg=>0.14583333333333334, :write_ms_max=>56, :compute_ms_avg=>10, :records_read=>6, :shuffle_output_bytes=>270, :read_ratio_avg=>0, :wait_ratio_avg=>0.2760416666666667}, {:wait_ms_max=>192, :wait_ratio_max=>1, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>148, :wait_ms_avg=>192, :write_ratio_max=>0.7708333333333334, :completed_parallel_inputs=>1, :records_written=>6, :id=>6, :compute_ratio_avg=>0.08333333333333333, :read_ms_max=>0, :input_stages=>["5"], :parallel_inputs=>1, :compute_ratio_max=>0.08333333333333333, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$20, $21, $22, $23, $24, $25, $26, $27, $28", "FROM __stage05_output"]}, {:kind=>"WRITE", :substeps=>["$20, $21, $22, $23, $24, $25, $26, $27", "TO __stage06_output"]}], :status=>"COMPLETE", :compute_ms_max=>16, :end_ms=>1721386862687, :name=>"S06: Output", :start_ms=>1721386862493, :write_ratio_avg=>0.7708333333333334, :write_ms_max=>148, :compute_ms_avg=>16, :records_read=>6, :shuffle_output_bytes=>0, :read_ratio_avg=>0, :wait_ratio_avg=>1}], :total_bytes_billed=>20971520, :estimated_bytes_processed=>232, :total_slot_ms=>27779, :total_bytes_processed=>232, :total_partitions_processed=>0, :cache_hit=>false, :timeline=>[{:total_slot_ms=>16975, :pending_units=>1, :elapsed_ms=>721, :active_units=>1, :completed_units=>101}, {:total_slot_ms=>27779, :pending_units=>0, :elapsed_ms=>1506, :active_units=>50, :completed_units=>155}], :statement_type=>"MERGE", :num_dml_affected_rows=>6}}
2024-07-19 11:01:11.674 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Get table... systemn-playground:test_by_sano.output
2024-07-19 11:01:12.481 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Delete table... systemn-playground:test_by_sano.LOAD_TEMP_a1848ee6_034f_4929_9f57_66ef4226c465_output
2024-07-19 11:01:12.793 +0000 [INFO] (0001:transaction): embulk-output-bigquery: delete /tmp/embulk_output_bigquery_20240719-17924-gnor1d.17924.2004.jsonl.gz
2024-07-19 11:01:12.801 +0000 [INFO] (main): Committed.
2024-07-19 11:01:12.801 +0000 [INFO] (main): Next config diff: {"in":{"last_path":"test_1.jsonl"},"out":{}}

$ embulk run config_in.yml
2024-07-19 11:01:20.009 +0000: Embulk v0.9.26
2024-07-19 11:01:20.622 +0000 [WARN] (main): DEPRECATION: JRuby org.jruby.embed.ScriptingContainer is directly injected.
2024-07-19 11:01:21.872 +0000 [INFO] (main): Gem's home and path are set by default: "/home/ubuntu/.embulk/lib/gems"
2024-07-19 11:01:22.408 +0000 [INFO] (main): Started Embulk v0.9.26
WARNING: You are running Ruby 2.3.3, which is nearing end-of-life.
The Google Cloud API clients work best on supported versions of Ruby. Consider upgrading to Ruby 2.4 or later.
See https://www.ruby-lang.org/en/downloads/branches/ for more info on the Ruby maintenance schedule.
To suppress this message, set the GOOGLE_CLOUD_SUPPRESS_RUBY_WARNINGS environment variable.
2024-07-19 11:01:25.657 +0000 [INFO] (0001:transaction): Loaded plugin embulk-input-bigquery (0.1.0)
2024-07-19 11:01:25.688 +0000 [INFO] (0001:transaction): Loaded plugin embulk-output-command (0.1.4)
2024-07-19 11:01:26.318 +0000 [INFO] (0001:transaction): determine columns using the getQueryResults API instead of the config.yml
2024-07-19 11:01:27.321 +0000 [INFO] (0001:transaction): waiting for the query job to complete to get schema from query results
2024-07-19 11:01:27.321 +0000 [INFO] (0001:transaction): completed: job_id=job_dMIRGnnRLlF-8LMM8JvB754xpmJj
2024-07-19 11:01:28.094 +0000 [INFO] (0001:transaction): determined columns: [{"name"=>"key_long", "type"=>:long}, {"name"=>"key_string", "type"=>:string}, {"name"=>"test_boolean", "type"=>:boolean}, {"name"=>"test_long", "type"=>:long}, {"name"=>"test_double", "type"=>:double}, {"name"=>"test_string", "type"=>:string}, {"name"=>"test_timestamp", "type"=>:timestamp}, {"name"=>"test_json", "type"=>:string}]
2024-07-19 11:01:28.100 +0000 [INFO] (0001:transaction): Using local thread executor with max_threads=16 / output tasks 8 = input tasks 1 * 8
2024-07-19 11:01:28.136 +0000 [INFO] (0001:transaction): Loaded plugin embulk-formatter-jsonl (0.1.4)
2024-07-19 11:01:28.150 +0000 [INFO] (0001:transaction): {done:  0 / 1, running: 0}
2024-07-19 11:01:28.158 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:01:28.168 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:01:28.170 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:01:28.172 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:01:28.174 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:01:28.175 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:01:28.178 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:01:28.180 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
{"key_long":1,"key_string":"a","test_boolean":true,"test_long":123,"test_double":1.23,"test_string":"あいうえお","test_timestamp":"1999-12-31 23:59:59 +0000","test_json":"{\"キー\":\"値\"}"}
{"key_long":2,"key_string":"b","test_boolean":false,"test_long":456,"test_double":4.56,"test_string":"かきくけこ","test_timestamp":"2000-01-01 00:00:00 +0000","test_json":"[{\"キー1\":\"値1\"},{\"キー2\":\"値2\"}]"}
{"key_long":3,"key_string":"c","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":null}
{"key_long":4,"key_string":"d","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":"12345"}
{"key_long":5,"key_string":"e","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":"123.45"}
{"key_long":6,"key_string":"f","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":"\"hoge\""}
2024-07-19 11:01:29.412 +0000 [INFO] (0001:transaction): {done:  1 / 1, running: 0}
2024-07-19 11:01:29.419 +0000 [INFO] (main): Committed.
2024-07-19 11:01:29.419 +0000 [INFO] (main): Next config diff: {"in":{},"out":{}}

run test 2

$ embulk run config_out.yml
2024-07-19 11:02:38.293 +0000: Embulk v0.9.26
2024-07-19 11:02:38.906 +0000 [WARN] (main): DEPRECATION: JRuby org.jruby.embed.ScriptingContainer is directly injected.
2024-07-19 11:02:40.319 +0000 [INFO] (main): Gem's home and path are set by default: "/home/ubuntu/.embulk/lib/gems"
2024-07-19 11:02:40.933 +0000 [INFO] (main): Started Embulk v0.9.26
2024-07-19 11:02:43.562 +0000 [INFO] (0001:transaction): Loaded plugin embulk-output-bigquery (0.7.1)
2024-07-19 11:02:43.609 +0000 [INFO] (0001:transaction): Loaded plugin embulk-parser-jsonl (0.2.1)
2024-07-19 11:02:43.620 +0000 [INFO] (0001:transaction): Listing local files at directory '.' filtering filename by prefix 'test_2.jsonl'
2024-07-19 11:02:43.621 +0000 [INFO] (0001:transaction): "follow_symlinks" is set false. Note that symbolic links to directories are skipped.
2024-07-19 11:02:43.625 +0000 [INFO] (0001:transaction): Loading files [test_2.jsonl]
2024-07-19 11:02:43.652 +0000 [INFO] (0001:transaction): Using local thread executor with max_threads=16 / output tasks 8 = input tasks 1 * 8
2024-07-19 11:02:43.691 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Get dataset... systemn-playground:test_by_sano
2024-07-19 11:02:44.963 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Create table... systemn-playground:test_by_sano.LOAD_TEMP_84c266bf_5f3d_4247_8573_96967aa73d3b_output
2024-07-19 11:02:45.386 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Create table... systemn-playground:test_by_sano.output
2024-07-19 11:02:45.796 +0000 [INFO] (0001:transaction): {done:  0 / 1, running: 0}
2024-07-19 11:02:45.886 +0000 [INFO] (embulk-output-executor-0): embulk-output-bigquery: create /tmp/embulk_output_bigquery_20240719-18028-15mh0ma.18028.2004.jsonl.gz
2024-07-19 11:02:45.915 +0000 [INFO] (0001:transaction): {done:  1 / 1, running: 0}
2024-07-19 11:02:45.923 +0000 [INFO] (Ruby-0-Thread-1: /home/ubuntu/.embulk/lib/gems/gems/embulk-output-bigquery-0.7.1/lib/embulk/output/bigquery/bigquery_client.rb:153): embulk-output-bigquery: Load job starting... job_id:[embulk_load_job_7462b213-c0a3-4c25-8c0f-b9bbc73dde51] /tmp/embulk_output_bigquery_20240719-18028-15mh0ma.18028.2004.jsonl.gz => systemn-playground:test_by_sano.LOAD_TEMP_84c266bf_5f3d_4247_8573_96967aa73d3b_output in US
2024-07-19 11:02:48.646 +0000 [INFO] (Ruby-0-Thread-1: /home/ubuntu/.embulk/lib/gems/gems/embulk-output-bigquery-0.7.1/lib/embulk/output/bigquery/bigquery_client.rb:153): embulk-output-bigquery: Load job checking... job_id:[embulk_load_job_7462b213-c0a3-4c25-8c0f-b9bbc73dde51] elapsed_time:4.8e-05sec status:[RUNNING]
2024-07-19 11:02:58.929 +0000 [INFO] (Ruby-0-Thread-1: /home/ubuntu/.embulk/lib/gems/gems/embulk-output-bigquery-0.7.1/lib/embulk/output/bigquery/bigquery_client.rb:153): embulk-output-bigquery: Load job completed... job_id:[embulk_load_job_7462b213-c0a3-4c25-8c0f-b9bbc73dde51] elapsed_time:10.284191sec status:[DONE]
2024-07-19 11:02:58.930 +0000 [INFO] (Ruby-0-Thread-1: /home/ubuntu/.embulk/lib/gems/gems/embulk-output-bigquery-0.7.1/lib/embulk/output/bigquery/bigquery_client.rb:153): embulk-output-bigquery: Load job response... job_id:[embulk_load_job_7462b213-c0a3-4c25-8c0f-b9bbc73dde51] response.statistics:{:start_time=>1721386968460, :completion_ratio=>1, :creation_time=>1721386968228, :end_time=>1721386969826, :reservation_id=>"default-pipeline", :total_slot_ms=>486, :load=>{:input_file_bytes=>1039, :output_bytes=>232, :output_rows=>6, :bad_records=>0, :input_files=>1}}
2024-07-19 11:02:58.933 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Get table... systemn-playground:test_by_sano.LOAD_TEMP_84c266bf_5f3d_4247_8573_96967aa73d3b_output
2024-07-19 11:02:59.287 +0000 [INFO] (0001:transaction): embulk-output-bigquery: transaction_report: {"num_input_rows":6,"num_response_rows":6,"num_output_rows":6,"num_rejected_rows":0}
2024-07-19 11:02:59.288 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job starting... job_id:[embulk_query_job_6861efa3-605d-4941-9ff6-fe87076e367f]
2024-07-19 11:03:00.946 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job checking... job_id:[embulk_query_job_6861efa3-605d-4941-9ff6-fe87076e367f] elapsed_time:4.9e-05sec status:[RUNNING]
2024-07-19 11:03:11.235 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job completed... job_id:[embulk_query_job_6861efa3-605d-4941-9ff6-fe87076e367f] elapsed_time:10.289786999999999sec status:[DONE]
2024-07-19 11:03:11.243 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job response... job_id:[embulk_query_job_6861efa3-605d-4941-9ff6-fe87076e367f] response.statistics:{:start_time=>1721386979599, :creation_time=>1721386979477, :end_time=>1721386980921, :total_slot_ms=>62023, :total_bytes_processed=>20971520, :query=>{:billing_tier=>1, :referenced_tables=>[{:project_id=>"systemn-playground", :dataset_id=>"test_by_sano", :table_id=>"INFORMATION_SCHEMA.TABLE_CONSTRAINTS"}, {:project_id=>"systemn-playground", :dataset_id=>"test_by_sano", :table_id=>"INFORMATION_SCHEMA.TABLE_CONSTRAINTS"}, {:project_id=>"systemn-playground", :dataset_id=>"test_by_sano", :table_id=>"INFORMATION_SCHEMA.KEY_COLUMN_USAGE"}, {:project_id=>"systemn-playground", :dataset_id=>"test_by_sano", :table_id=>"INFORMATION_SCHEMA.KEY_COLUMN_USAGE"}], :query_plan=>[{:wait_ms_max=>229, :wait_ratio_max=>1, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>4, :wait_ms_avg=>141, :write_ratio_max=>0.056768558951965066, :completed_parallel_inputs=>15, :records_written=>1, :id=>0, :compute_ratio_avg=>0.07860262008733625, :read_ms_max=>0, :parallel_inputs=>15, :compute_ratio_max=>0.10043668122270742, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$10:pk_uuid", "FROM systemn-playground.region-US.INFORMATION_SCHEMA.TABLE_CONSTRAINTS_REFERENCED_DATASET_CONSTRAINTS_TABLE"]}, {:kind=>"WRITE", :substeps=>["$10", "TO __stage00_output"]}], :status=>"COMPLETE", :compute_ms_max=>23, :end_ms=>1721386980002, :name=>"S00: Input", :start_ms=>1721386979721, :write_ratio_avg=>0.017467248908296942, :write_ms_max=>13, :compute_ms_avg=>18, :records_read=>1, :shuffle_output_bytes=>41, :read_ratio_avg=>0, :wait_ratio_avg=>0.6157205240174672}, {:wait_ms_max=>222, :wait_ratio_max=>0.9694323144104804, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>4, :wait_ms_avg=>153, :write_ratio_max=>0.039301310043668124, :completed_parallel_inputs=>15, :records_written=>1, :id=>1, :compute_ratio_avg=>0.07423580786026202, :read_ms_max=>0, :parallel_inputs=>15, :compute_ratio_max=>0.09606986899563319, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$1:pk_uuid", "FROM systemn-playground.region-US.INFORMATION_SCHEMA.TABLE_CONSTRAINTS_REFERENCED_DATASET_CONSTRAINTS_TABLE"]}, {:kind=>"WRITE", :substeps=>["$1", "TO __stage01_output"]}], :status=>"COMPLETE", :compute_ms_max=>22, :end_ms=>1721386980004, :name=>"S01: Input", :start_ms=>1721386979757, :write_ratio_avg=>0.017467248908296942, :write_ms_max=>9, :compute_ms_avg=>17, :records_read=>1, :shuffle_output_bytes=>41, :read_ratio_avg=>0, :wait_ratio_avg=>0.6681222707423581}, {:wait_ms_max=>82, :wait_ratio_max=>0.35807860262008734, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>5, :wait_ms_avg=>44, :write_ratio_max=>0.25327510917030566, :completed_parallel_inputs=>100, :records_written=>1, :id=>2, :compute_ratio_avg=>0.034934497816593885, :read_ms_max=>0, :input_stages=>["0"], :parallel_inputs=>100, :compute_ratio_max=>0.0611353711790393, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["FROM __stage00_output"]}], :status=>"COMPLETE", :compute_ms_max=>14, :end_ms=>1721386980194, :name=>"S02: Coalesce", :start_ms=>1721386980005, :write_ratio_avg=>0.021834061135371178, :write_ms_max=>58, :compute_ms_avg=>8, :records_read=>1, :shuffle_output_bytes=>41, :read_ratio_avg=>0, :wait_ratio_avg=>0.19213973799126638}, {:wait_ms_max=>71, :wait_ratio_max=>0.31004366812227074, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>7, :wait_ms_avg=>71, :write_ratio_max=>0.03056768558951965, :completed_parallel_inputs=>1, :records_written=>1, :id=>3, :compute_ratio_avg=>0.10043668122270742, :read_ms_max=>0, :input_stages=>["2"], :parallel_inputs=>1, :compute_ratio_max=>0.10043668122270742, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$30:constraint_catalog, $31:constraint_schema, $32:constraint_name, $33:table_catalog, $34:table_schema, $35:table_name, $37:pk_uuid, $36:constraint_type", "FROM systemn-playground.test_by_sano.INFORMATION_SCHEMA.TABLE_CONSTRAINTS_REFERENCING_DATASET_CONSTRAINTS_TABLE", "WHERE and(equal($35, 'output'), equal($36, 'PRIMARY KEY'))"]}, {:kind=>"READ", :substeps=>["$10", "FROM __stage02_output"]}, {:kind=>"JOIN", :substeps=>["$70 := $30, $71 := $31, $72 := $32, $73 := $33, $74 := $34, $75 := $35", "SEMI HASH JOIN EACH  WITH ALL  ON $37 = $10"]}, {:kind=>"WRITE", :substeps=>["$70, $71, $72, $73, $74, $75", "TO __stage03_output"]}], :status=>"COMPLETE", :compute_ms_max=>23, :end_ms=>1721386980328, :name=>"S03: Join", :start_ms=>1721386980094, :write_ratio_avg=>0.03056768558951965, :write_ms_max=>7, :compute_ms_avg=>23, :records_read=>2, :shuffle_output_bytes=>106, :read_ratio_avg=>0, :wait_ratio_avg=>0.31004366812227074}, {:wait_ms_max=>96, :wait_ratio_max=>0.4192139737991266, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>3, :wait_ms_avg=>62, :write_ratio_max=>0.22270742358078602, :completed_parallel_inputs=>100, :records_written=>1, :id=>4, :compute_ratio_avg=>0.034934497816593885, :read_ms_max=>0, :input_stages=>["1"], :parallel_inputs=>100, :compute_ratio_max=>0.0611353711790393, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["FROM __stage01_output"]}], :status=>"COMPLETE", :compute_ms_max=>14, :end_ms=>1721386980190, :name=>"S04: Coalesce", :start_ms=>1721386980021, :write_ratio_avg=>0.013100436681222707, :write_ms_max=>51, :compute_ms_avg=>8, :records_read=>1, :shuffle_output_bytes=>41, :read_ratio_avg=>0, :wait_ratio_avg=>0.27074235807860264}, {:wait_ms_max=>62, :wait_ratio_max=>0.27074235807860264, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>12, :wait_ms_avg=>21, :write_ratio_max=>0.2445414847161572, :completed_parallel_inputs=>100, :records_written=>1, :id=>6, :compute_ratio_avg=>0.034934497816593885, :read_ms_max=>0, :input_stages=>["3"], :parallel_inputs=>100, :compute_ratio_max=>0.06550218340611354, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["FROM __stage03_output"]}], :status=>"COMPLETE", :compute_ms_max=>15, :end_ms=>1721386980541, :name=>"S06: Coalesce", :start_ms=>1721386980329, :write_ratio_avg=>0.05240174672489083, :write_ms_max=>56, :compute_ms_avg=>8, :records_read=>1, :shuffle_output_bytes=>106, :read_ratio_avg=>0, :wait_ratio_avg=>0.09170305676855896}, {:wait_ms_max=>71, :wait_ratio_max=>0.31004366812227074, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>8, :wait_ms_avg=>71, :write_ratio_max=>0.034934497816593885, :completed_parallel_inputs=>1, :records_written=>2, :id=>7, :compute_ratio_avg=>0.1091703056768559, :read_ms_max=>0, :input_stages=>["6", "4"], :parallel_inputs=>1, :compute_ratio_max=>0.1091703056768559, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$20:constraint_catalog, $21:constraint_schema, $22:constraint_name, $23:table_catalog, $24:table_schema, $25:table_name, $26:column_name, $27:ordinal_position, $28:pk_uuid", "FROM systemn-playground.test_by_sano.INFORMATION_SCHEMA.KEY_COLUMN_USAGE_DATASET_KEY_COLUMN_USAGE_TABLE", "WHERE equal($25, 'output')"]}, {:kind=>"READ", :substeps=>["$1", "FROM __stage04_output"]}, {:kind=>"READ", :substeps=>["$70, $71, $72, $73, $74, $75", "FROM __stage06_output"]}, {:kind=>"JOIN", :substeps=>["$80 := $56, $81 := $57", "INNER HASH JOIN EACH  WITH ALL  ON $50 = $70, $51 = $71, $52 = $72, $53 = $73, $54 = $74, $55 = $75"]}, {:kind=>"JOIN", :substeps=>["$50 := $20, $51 := $21, $52 := $22, $53 := $23, $54 := $24, $55 := $25, $56 := $26, $57 := $27", "SEMI HASH JOIN EACH  WITH ALL  ON $28 = $1"]}, {:kind=>"WRITE", :substeps=>["$80, $81", "TO __stage07_output"]}], :status=>"COMPLETE", :compute_ms_max=>25, :end_ms=>1721386980683, :name=>"S07: Join", :start_ms=>1721386980395, :write_ratio_avg=>0.034934497816593885, :write_ms_max=>8, :compute_ms_avg=>25, :records_read=>4, :shuffle_output_bytes=>46, :read_ratio_avg=>0, :wait_ratio_avg=>0.31004366812227074}, {:wait_ms_max=>16, :wait_ratio_max=>0.06986899563318777, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>0, :wait_ms_avg=>16, :write_ratio_max=>0, :completed_parallel_inputs=>1, :records_written=>2, :id=>8, :compute_ratio_avg=>0.017467248908296942, :read_ms_max=>0, :input_stages=>["7"], :parallel_inputs=>1, :compute_ratio_max=>0.017467248908296942, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$80, $81", "FROM __stage07_output"]}, {:kind=>"SORT", :substeps=>["$81 ASC"]}, {:kind=>"WRITE", :substeps=>["$90", "TO __stage08_output"]}], :status=>"COMPLETE", :compute_ms_max=>4, :end_ms=>1721386980712, :name=>"S08: Output", :start_ms=>1721386980705, :write_ratio_avg=>0, :write_ms_max=>0, :compute_ms_avg=>4, :records_read=>2, :shuffle_output_bytes=>28, :read_ratio_avg=>0, :wait_ratio_avg=>0.06986899563318777}], :total_bytes_billed=>20971520, :estimated_bytes_processed=>20971520, :total_slot_ms=>62023, :total_bytes_processed=>20971520, :total_partitions_processed=>0, :cache_hit=>false, :timeline=>[{:total_slot_ms=>35346, :pending_units=>2, :elapsed_ms=>614, :active_units=>2, :completed_units=>230}, {:total_slot_ms=>62023, :pending_units=>0, :elapsed_ms=>1119, :active_units=>0, :completed_units=>333}], :statement_type=>"SELECT"}}
2024-07-19 11:03:11.661 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Execute query... MERGE `test_by_sano`.`output` T USING `test_by_sano`.`LOAD_TEMP_84c266bf_5f3d_4247_8573_96967aa73d3b_output` S ON T.`key_long` = S.`key_long` AND T.`key_string` = S.`key_string` WHEN MATCHED THEN UPDATE SET T.`key_long` = S.`key_long`, T.`key_string` = S.`key_string`, T.`test_boolean` = S.`test_boolean`, T.`test_long` = S.`test_long`, T.`test_double` = S.`test_double`, T.`test_string` = S.`test_string`, T.`test_timestamp` = S.`test_timestamp`, T.`test_json` = S.`test_json` WHEN NOT MATCHED THEN INSERT (`key_long`, `key_string`, `test_boolean`, `test_long`, `test_double`, `test_string`, `test_timestamp`, `test_json`) VALUES (`key_long`, `key_string`, `test_boolean`, `test_long`, `test_double`, `test_string`, `test_timestamp`, `test_json`)
2024-07-19 11:03:11.662 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job starting... job_id:[embulk_query_job_e80e2cab-3c9a-4868-9ee3-48e987fe650d]
2024-07-19 11:03:12.592 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job checking... job_id:[embulk_query_job_e80e2cab-3c9a-4868-9ee3-48e987fe650d] elapsed_time:4.6e-05sec status:[RUNNING]
2024-07-19 11:03:22.877 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job completed... job_id:[embulk_query_job_e80e2cab-3c9a-4868-9ee3-48e987fe650d] elapsed_time:10.284484sec status:[DONE]
2024-07-19 11:03:22.882 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Query job response... job_id:[embulk_query_job_e80e2cab-3c9a-4868-9ee3-48e987fe650d] response.statistics:{:start_time=>1721386992444, :creation_time=>1721386991871, :end_time=>1721386994952, :total_slot_ms=>32406, :total_bytes_processed=>464, :query=>{:billing_tier=>1, :referenced_tables=>[{:project_id=>"systemn-playground", :dataset_id=>"test_by_sano", :table_id=>"LOAD_TEMP_84c266bf_5f3d_4247_8573_96967aa73d3b_output"}, {:project_id=>"systemn-playground", :dataset_id=>"test_by_sano", :table_id=>"output"}], :query_plan=>[{:wait_ms_max=>16, :wait_ratio_max=>0.06666666666666667, :read_ms_avg=>5, :read_ratio_max=>0.020833333333333332, :write_ms_avg=>18, :wait_ms_avg=>16, :write_ratio_max=>0.075, :completed_parallel_inputs=>1, :records_written=>6, :id=>0, :compute_ratio_avg=>0.058333333333333334, :read_ms_max=>5, :parallel_inputs=>1, :compute_ratio_max=>0.058333333333333334, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$10:key_long, $11:key_string, $12:$file_temp_id, $13:$row_temp_id", "FROM test_by_sano.output"]}, {:kind=>"COMPUTE", :substeps=>["$150 := 1"]}, {:kind=>"WRITE", :substeps=>["$10, $11, $12, $13, $150", "TO __stage00_output"]}], :status=>"COMPLETE", :compute_ms_max=>14, :end_ms=>1721386992947, :name=>"S00: Input", :start_ms=>1721386992748, :write_ratio_avg=>0.075, :write_ms_max=>18, :compute_ms_avg=>14, :records_read=>6, :shuffle_output_bytes=>252, :read_ratio_avg=>0.020833333333333332, :wait_ratio_avg=>0.06666666666666667}, {:wait_ms_max=>37, :wait_ratio_max=>0.15416666666666667, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>5, :wait_ms_avg=>22, :write_ratio_max=>0.2791666666666667, :completed_parallel_inputs=>100, :records_written=>6, :id=>1, :compute_ratio_avg=>0.0375, :read_ms_max=>0, :input_stages=>["0"], :parallel_inputs=>100, :compute_ratio_max=>0.0625, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["FROM __stage00_output"]}], :status=>"COMPLETE", :compute_ms_max=>15, :end_ms=>1721386993243, :name=>"S01: Coalesce", :start_ms=>1721386992955, :write_ratio_avg=>0.020833333333333332, :write_ms_max=>67, :compute_ms_avg=>9, :records_read=>6, :shuffle_output_bytes=>252, :read_ratio_avg=>0, :wait_ratio_avg=>0.09166666666666666}, {:wait_ms_max=>51, :wait_ratio_max=>0.2125, :read_ms_avg=>3, :read_ratio_max=>0.0125, :write_ms_avg=>3, :wait_ms_avg=>51, :write_ratio_max=>0.0125, :completed_parallel_inputs=>1, :records_written=>6, :id=>2, :compute_ratio_avg=>0.07083333333333333, :read_ms_max=>3, :input_stages=>["1"], :parallel_inputs=>1, :compute_ratio_max=>0.07083333333333333, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$1:key_long, $2:key_string, $3:test_boolean, $4:test_long, $5:test_double, $6:test_string, $7:test_timestamp, $8:test_json", "FROM test_by_sano.LOAD_TEMP_84c266bf_5f3d_4247_8573_96967aa73d3b_output"]}, {:kind=>"READ", :substeps=>["$10, $11, $12, $13, $150", "FROM __stage01_output"]}, {:kind=>"AGGREGATE", :substeps=>["GROUP BY $180 := $101, $181 := $102, $182 := $90", "$70 := ANY_AND_CHECK_FOR_UPDATE($121)", "$71 := ANY_AND_CHECK_FOR_UPDATE($120)", "$72 := ANY_AND_CHECK_FOR_UPDATE($122)", "$73 := ANY_AND_CHECK_FOR_UPDATE($123)", "$74 := ANY_AND_CHECK_FOR_UPDATE($124)", "$75 := ANY_AND_CHECK_FOR_UPDATE($125)", "$76 := ANY_AND_CHECK_FOR_UPDATE($126)", "$77 := ANY_AND_CHECK_FOR_UPDATE($127)", "$78 := ANY_AND_CHECK_FOR_UPDATE($128)", "$79 := ANY_AND_CHECK_FOR_UPDATE($129)", "$80 := ANY_AND_CHECK_FOR_UPDATE($141)", "$81 := ANY_AND_CHECK_FOR_UPDATE($142)", "$82 := ANY_AND_CHECK_FOR_UPDATE($143)", "$83 := ANY_AND_CHECK_FOR_UPDATE($144)", "$84 := ANY_AND_CHECK_FOR_UPDATE($145)", "$85 := ANY_AND_CHECK_FOR_UPDATE($146)"]}, {:kind=>"COMPUTE", :substeps=>["$90 := if(equal($120, 10001), $100, NULL)"]}, {:kind=>"COMPUTE", :substeps=>["$100 := UNIQUE_ROW_ID()", "$101 := if(equal($120, 10001), SAFE_EXPR(CAST(add(1000000000, abs(mod(farm_fingerprint($110), 1000000000))) AS UINT64)), $160)", "$102 := if(equal($120, 10001), 18446744073709551615, $161)"]}, {:kind=>"COMPUTE", :substeps=>["$110 := UNIQUE_ROW_ID()"]}, {:kind=>"FILTER", :substeps=>["not(equal($120, 0))"]}, {:kind=>"COMPUTE", :substeps=>["$120 := if($130, 10001, $140)", "$121 := if($130, 2, if(not(is_null($162)), 1, -1))", "$122 := if($130, $163, NULL)", "$123 := if($130, $164, NULL)", "$124 := if($130, $165, NULL)", "$125 := if($130, $166, NULL)", "$126 := if($130, $167, NULL)", "$127 := if($130, $168, NULL)", "$128 := if($130, $169, NULL)", "$129 := if($130, $170, NULL)"]}, {:kind=>"COMPUTE", :substeps=>["$130 := and(equal($140, 0), is_null($162))"]}, {:kind=>"COMPUTE", :substeps=>["$140 := if(not(is_null($162)), 10002, 0)", "$141 := if(not(is_null($162)), $163, NULL)", "$142 := if(not(is_null($162)), $164, NULL)", "$143 := if(not(is_null($162)), $165, NULL)", "$144 := if(not(is_null($162)), $166, NULL)", "$145 := if(not(is_null($162)), $167, NULL)", "$146 := if(not(is_null($162)), $168, NULL)", "$147 := if(not(is_null($162)), $169, NULL)", "$148 := if(not(is_null($162)), $170, NULL)"]}, {:kind=>"JOIN", :substeps=>["$163 := $1, $164 := $2, $165 := $3, $166 := $4, $167 := $5, $168 := $6, $169 := $7, $170 := $8, $160 := $12, $161 := $13, $162 := $150", "LEFT OUTER HASH JOIN EACH  WITH ALL  ON $1 = $10, $2 = $11"]}, {:kind=>"WRITE", :substeps=>["$70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85", "TO __stage02_output", "BY HASH($180, $181, $182)"]}], :status=>"COMPLETE", :compute_ms_max=>17, :end_ms=>1721386993338, :name=>"S02: Join+", :start_ms=>1721386992993, :write_ratio_avg=>0.0125, :write_ms_max=>3, :compute_ms_avg=>17, :records_read=>12, :shuffle_output_bytes=>666, :read_ratio_avg=>0.0125, :wait_ratio_avg=>0.2125}, {:wait_ms_max=>21, :wait_ratio_max=>0.0875, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>4, :wait_ms_avg=>21, :write_ratio_max=>0.016666666666666666, :completed_parallel_inputs=>1, :records_written=>6, :id=>3, :compute_ratio_avg=>0.020833333333333332, :read_ms_max=>0, :input_stages=>["2"], :parallel_inputs=>1, :compute_ratio_max=>0.020833333333333332, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$70, $71, $72, $73, $74, $75, $76, $77, $78, $79, $80, $81, $82, $83, $84, $85", "FROM __stage02_output"]}, {:kind=>"AGGREGATE", :substeps=>["GROUP BY $190 := $180, $191 := $181, $192 := $182", "$50 := ANY_AND_CHECK_FOR_UPDATE($70)", "$51 := ANY_AND_CHECK_FOR_UPDATE($71)", "$52 := ANY_AND_CHECK_FOR_UPDATE($72)", "$53 := ANY_AND_CHECK_FOR_UPDATE($73)", "$54 := ANY_AND_CHECK_FOR_UPDATE($74)", "$55 := ANY_AND_CHECK_FOR_UPDATE($75)", "$56 := ANY_AND_CHECK_FOR_UPDATE($76)", "$57 := ANY_AND_CHECK_FOR_UPDATE($77)", "$58 := ANY_AND_CHECK_FOR_UPDATE($78)", "$59 := ANY_AND_CHECK_FOR_UPDATE($79)", "$60 := ANY_AND_CHECK_FOR_UPDATE($80)", "$61 := ANY_AND_CHECK_FOR_UPDATE($81)", "$62 := ANY_AND_CHECK_FOR_UPDATE($82)", "$63 := ANY_AND_CHECK_FOR_UPDATE($83)", "$64 := ANY_AND_CHECK_FOR_UPDATE($84)", "$65 := ANY_AND_CHECK_FOR_UPDATE($85)"]}, {:kind=>"WRITE", :substeps=>["$50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, $64, $65", "TO __stage03_output", "BY HASH($190)"]}], :status=>"COMPLETE", :compute_ms_max=>5, :end_ms=>1721386993381, :name=>"S03: Aggregate", :start_ms=>1721386993368, :write_ratio_avg=>0.016666666666666666, :write_ms_max=>4, :compute_ms_avg=>5, :records_read=>6, :shuffle_output_bytes=>570, :read_ratio_avg=>0, :wait_ratio_avg=>0.0875}, {:wait_ms_max=>17, :wait_ratio_max=>0.07083333333333333, :read_ms_avg=>16, :read_ratio_max=>0.06666666666666667, :write_ms_avg=>36, :wait_ms_avg=>17, :write_ratio_max=>0.15, :completed_parallel_inputs=>1, :records_written=>9, :id=>4, :compute_ratio_avg=>0.04583333333333333, :read_ms_max=>16, :input_stages=>["3"], :parallel_inputs=>1, :compute_ratio_max=>0.04583333333333333, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, $64, $65", "FROM __stage03_output"]}, {:kind=>"COMPUTE", :substeps=>["$20 := if(equal(2, $30), $221, if(equal(1, $30), $229, $40))", "$21 := if(equal(2, $30), $222, if(equal(1, $30), $230, $41))", "$22 := if(equal(2, $30), $223, if(equal(1, $30), $231, $42))", "$23 := if(equal(2, $30), $224, if(equal(1, $30), $232, $43))", "$24 := if(equal(2, $30), $225, if(equal(1, $30), $233, $44))", "$25 := if(equal(2, $30), $226, if(equal(1, $30), $234, $45))", "$26 := if(equal(2, $30), $227, if(equal(1, $30), $235, $46))", "$27 := if(equal(2, $30), $228, if(equal(1, $30), $236, $47))", "$28 := if(equal(-1, $30), 'REINSERT', if(in($30, 1), 'UPDATE', NULL))"]}, {:kind=>"COMPUTE", :substeps=>["$30 := if(not($48), -1, $220)"]}, {:kind=>"SORT", :substeps=>["$190 ASC, $191 DESC"]}, {:kind=>"WRITE", :substeps=>["$20, $21, $22, $23, $24, $25, $26, $27, $28", "TO __stage04_output"]}], :status=>"COMPLETE", :compute_ms_max=>11, :end_ms=>1721386993529, :name=>"S04: Sort+", :start_ms=>1721386993423, :write_ratio_avg=>0.15, :write_ms_max=>36, :compute_ms_avg=>11, :records_read=>12, :shuffle_output_bytes=>487, :read_ratio_avg=>0.06666666666666667, :wait_ratio_avg=>0.07083333333333333}, {:wait_ms_max=>19, :wait_ratio_max=>0.07916666666666666, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>25, :wait_ms_avg=>16, :write_ratio_max=>0.3, :completed_parallel_inputs=>50, :records_written=>9, :id=>5, :compute_ratio_avg=>0.0375, :read_ms_max=>0, :input_stages=>["4"], :parallel_inputs=>50, :compute_ratio_max=>0.0625, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["FROM __stage04_output"]}], :status=>"COMPLETE", :compute_ms_max=>15, :end_ms=>1721386993677, :name=>"S05: Coalesce", :start_ms=>1721386993548, :write_ratio_avg=>0.10416666666666667, :write_ms_max=>72, :compute_ms_avg=>9, :records_read=>9, :shuffle_output_bytes=>487, :read_ratio_avg=>0, :wait_ratio_avg=>0.06666666666666667}, {:wait_ms_max=>240, :wait_ratio_max=>1, :read_ms_avg=>0, :read_ratio_max=>0, :write_ms_avg=>134, :wait_ms_avg=>240, :write_ratio_max=>0.5583333333333333, :completed_parallel_inputs=>1, :records_written=>9, :id=>6, :compute_ratio_avg=>0.0625, :read_ms_max=>0, :input_stages=>["5"], :parallel_inputs=>1, :compute_ratio_max=>0.0625, :shuffle_output_bytes_spilled=>0, :steps=>[{:kind=>"READ", :substeps=>["$20, $21, $22, $23, $24, $25, $26, $27, $28", "FROM __stage05_output"]}, {:kind=>"WRITE", :substeps=>["$20, $21, $22, $23, $24, $25, $26, $27", "TO __stage06_output"]}], :status=>"COMPLETE", :compute_ms_max=>15, :end_ms=>1721386993918, :name=>"S06: Output", :start_ms=>1721386993775, :write_ratio_avg=>0.5583333333333333, :write_ms_max=>134, :compute_ms_avg=>15, :records_read=>9, :shuffle_output_bytes=>0, :read_ratio_avg=>0, :wait_ratio_avg=>1}], :total_bytes_billed=>20971520, :estimated_bytes_processed=>464, :total_slot_ms=>32406, :total_bytes_processed=>464, :total_partitions_processed=>0, :cache_hit=>false, :timeline=>[{:total_slot_ms=>24027, :pending_units=>1, :elapsed_ms=>803, :active_units=>1, :completed_units=>101}, {:total_slot_ms=>32406, :pending_units=>0, :elapsed_ms=>1505, :active_units=>0, :completed_units=>155}], :statement_type=>"MERGE", :num_dml_affected_rows=>6}}
2024-07-19 11:03:22.884 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Get table... systemn-playground:test_by_sano.output
2024-07-19 11:03:23.678 +0000 [INFO] (0001:transaction): embulk-output-bigquery: Delete table... systemn-playground:test_by_sano.LOAD_TEMP_84c266bf_5f3d_4247_8573_96967aa73d3b_output
2024-07-19 11:03:23.995 +0000 [INFO] (0001:transaction): embulk-output-bigquery: delete /tmp/embulk_output_bigquery_20240719-18028-15mh0ma.18028.2004.jsonl.gz
2024-07-19 11:03:24.001 +0000 [INFO] (main): Committed.
2024-07-19 11:03:24.002 +0000 [INFO] (main): Next config diff: {"in":{"last_path":"test_2.jsonl"},"out":{}}

$ embulk run config_in.yml
2024-07-19 11:03:28.126 +0000: Embulk v0.9.26
2024-07-19 11:03:28.748 +0000 [WARN] (main): DEPRECATION: JRuby org.jruby.embed.ScriptingContainer is directly injected.
2024-07-19 11:03:30.068 +0000 [INFO] (main): Gem's home and path are set by default: "/home/ubuntu/.embulk/lib/gems"
2024-07-19 11:03:30.669 +0000 [INFO] (main): Started Embulk v0.9.26
WARNING: You are running Ruby 2.3.3, which is nearing end-of-life.
The Google Cloud API clients work best on supported versions of Ruby. Consider upgrading to Ruby 2.4 or later.
See https://www.ruby-lang.org/en/downloads/branches/ for more info on the Ruby maintenance schedule.
To suppress this message, set the GOOGLE_CLOUD_SUPPRESS_RUBY_WARNINGS environment variable.
2024-07-19 11:03:33.796 +0000 [INFO] (0001:transaction): Loaded plugin embulk-input-bigquery (0.1.0)
2024-07-19 11:03:33.826 +0000 [INFO] (0001:transaction): Loaded plugin embulk-output-command (0.1.4)
2024-07-19 11:03:34.456 +0000 [INFO] (0001:transaction): determine columns using the getQueryResults API instead of the config.yml
2024-07-19 11:03:35.426 +0000 [INFO] (0001:transaction): waiting for the query job to complete to get schema from query results
2024-07-19 11:03:35.426 +0000 [INFO] (0001:transaction): completed: job_id=job_iMSrpxyS45eYFP-HEAhmTcvPH6gW
2024-07-19 11:03:36.153 +0000 [INFO] (0001:transaction): determined columns: [{"name"=>"key_long", "type"=>:long}, {"name"=>"key_string", "type"=>:string}, {"name"=>"test_boolean", "type"=>:boolean}, {"name"=>"test_long", "type"=>:long}, {"name"=>"test_double", "type"=>:double}, {"name"=>"test_string", "type"=>:string}, {"name"=>"test_timestamp", "type"=>:timestamp}, {"name"=>"test_json", "type"=>:string}]
2024-07-19 11:03:36.159 +0000 [INFO] (0001:transaction): Using local thread executor with max_threads=16 / output tasks 8 = input tasks 1 * 8
2024-07-19 11:03:36.187 +0000 [INFO] (0001:transaction): Loaded plugin embulk-formatter-jsonl (0.1.4)
2024-07-19 11:03:36.200 +0000 [INFO] (0001:transaction): {done:  0 / 1, running: 0}
2024-07-19 11:03:36.208 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:03:36.217 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:03:36.220 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:03:36.221 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:03:36.223 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:03:36.225 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:03:36.228 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
2024-07-19 11:03:36.231 +0000 [INFO] (0023:task-0000): Using command [sh, -c, cat -]
{"key_long":1,"key_string":"a","test_boolean":true,"test_long":123,"test_double":1.23,"test_string":"あいうえお","test_timestamp":"1999-12-31 23:59:59 +0000","test_json":"{\"キー\":\"値\"}"}
{"key_long":2,"key_string":"b","test_boolean":false,"test_long":456,"test_double":4.56,"test_string":"かきくけこ","test_timestamp":"2000-01-01 00:00:00 +0000","test_json":"[{\"キー1\":\"値1\"},{\"キー2\":\"値2\"}]"}
{"key_long":3,"key_string":"c","test_boolean":false,"test_long":456,"test_double":4.56,"test_string":"かきくけこ","test_timestamp":"2000-01-01 00:00:00 +0000","test_json":"[{\"キー1\":\"値1\"},{\"キー2\":\"値2\"}]"}
{"key_long":4,"key_string":"d","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":"12345"}
{"key_long":5,"key_string":"e","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":null}
{"key_long":6,"key_string":"f","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":"\"hoge\""}
{"key_long":7,"key_string":"g","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":"12345"}
{"key_long":8,"key_string":"h","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":"123.45"}
{"key_long":9,"key_string":"i","test_boolean":null,"test_long":null,"test_double":null,"test_string":null,"test_timestamp":null,"test_json":"\"hoge\""}
2024-07-19 11:03:37.565 +0000 [INFO] (0001:transaction): {done:  1 / 1, running: 0}
2024-07-19 11:03:37.571 +0000 [INFO] (main): Committed.
2024-07-19 11:03:37.571 +0000 [INFO] (main): Next config diff: {"in":{},"out":{}}

[d-hrs]

LGTM