bug: Ipc error: Expected compressed length of 530512 got 530520

[wubx]

Search before asking

• I had searched in the issues and found no similar issues.

Version

v1.2.648-nightly

What's Wrong?

0f480d68-1143-4ea5-80f1-834f3e51eeb9 2024-10-23T14:55:41.505784+08:00 ERROR databend_common_tracing::panic_hook: panic_hook.rs:53 panicked at src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs:313:91:
called `Result::unwrap()` on an `Err` value: FromStdError. Code: 1001, Text = Ipc error: Expected compressed length of 530512 got 530520.

   0: databend_common_exception::exception_backtrace::capture
             at /workspace/src/common/exception/src/exception_backtrace.rs:82:17
   1: databend_common_exception::exception::ErrorCode<C>::from_std_error
             at /workspace/src/common/exception/src/exception.rs:291:24
   2: databend_common_exception::exception_into::<impl core::convert::From<arrow_schema::error::ArrowError> for databend_common_exception::exception::ErrorCode>::from
             at /workspace/src/common/exception/src/exception_into.rs:133:18
   3: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/result.rs:1989:27
      databend_common_expression::utils::arrow::read_column
             at /workspace/src/query/expression/src/utils/arrow.rs:108:15
   4: databend_common_expression::utils::arrow::deserialize_column
             at /workspace/src/query/expression/src/utils/arrow.rs:99:5
      databend_query::pipelines::processors::transforms::aggregator::serde::transform_spill_reader::TransformSpillReader<Method,V>::deserialize
             at /workspace/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs:313:26
   5: <databend_query::pipelines::processors::transforms::aggregator::serde::transform_spill_reader::TransformSpillReader<Method,V> as databend_common_pipeline_core::processors::processor::Processor>::process
             at /workspace/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs:158:47
   6: databend_common_pipeline_core::processors::processor::ProcessorPtr::process
             at /workspace/src/query/pipeline/core/src/processors/processor.rs:169:9
   7: databend_query::pipelines::executor::executor_worker_context::ExecutorWorkerContext::execute_sync_task
             at /workspace/src/query/service/src/pipelines/executor/executor_worker_context.rs:169:9
      databend_query::pipelines::executor::executor_worker_context::ExecutorWorkerContext::execute_task
             at /workspace/src/query/service/src/pipelines/executor/executor_worker_context.rs:132:52
   8: databend_query::pipelines::executor::query_pipeline_executor::QueryPipelineExecutor::execute_single_thread
             at /workspace/src/query/service/src/pipelines/executor/query_pipeline_executor.rs:406:35
   9: databend_query::pipelines::executor::query_pipeline_executor::QueryPipelineExecutor::execute_threads::{{closure}}::{{closure}}
             at /workspace/src/query/service/src/pipelines/executor/query_pipeline_executor.rs:378:50
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
  10: databend_common_base::runtime::catch_unwind::catch_unwind
             at /workspace/src/common/base/src/runtime/catch_unwind.rs:47:11
  11: databend_query::pipelines::executor::query_pipeline_executor::QueryPipelineExecutor::execute_threads::{{closure}}
             at /workspace/src/query/service/src/pipelines/executor/query_pipeline_executor.rs:378:34
      databend_common_base::runtime::runtime_tracker::ThreadTracker::tracking_function::{{closure}}::{{closure}}
             at /workspace/src/common/base/src/runtime/runtime_tracker.rs:208:17
      databend_common_base::runtime::thread::Thread::named_spawn::{{closure}}
             at /workspace/src/common/base/src/runtime/thread.rs:78:21
      std::sys::backtrace::__rust_begin_short_backtrace
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/backtrace.rs:155:18
  12: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/mod.rs:542:17
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
      std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/mod.rs:541:30
      core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/ops/function.rs:250:5
  13: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2064:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2064:9
      std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/pal/unix/thread.rs:108:17
  14: start_thread
             at ./nptl/pthread_create.c:442:8
  15: __GI___clone3
             at ./misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
 backtrace=   0: backtrace::backtrace::libunwind::trace
             at /opt/rust/cargo/git/checkouts/backtrace-rs-fb1f822361417489-shallow/72265be/src/backtrace/libunwind.rs:116:5
      backtrace::backtrace::trace_unsynchronized
             at /opt/rust/cargo/git/checkouts/backtrace-rs-fb1f822361417489-shallow/72265be/src/backtrace/mod.rs:66:5
      backtrace::backtrace::trace
             at /opt/rust/cargo/git/checkouts/backtrace-rs-fb1f822361417489-shallow/72265be/src/backtrace/mod.rs:53:14
   1: databend_common_tracing::panic_hook::captures_frames
             at /workspace/src/common/tracing/src/panic_hook.rs:66:5
      databend_common_tracing::panic_hook::backtrace
             at /workspace/src/common/tracing/src/panic_hook.rs:75:9
      databend_common_tracing::panic_hook::log_panic
             at /workspace/src/common/tracing/src/panic_hook.rs:47:25
   2: databend_common_tracing::panic_hook::set_panic_hook::{{closure}}
             at /workspace/src/common/tracing/src/panic_hook.rs:33:9
   3: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2078:9
      std::panicking::rust_panic_with_hook
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:804:13
   4: std::panicking::begin_panic_handler::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:670:13
   5: std::sys::backtrace::__rust_end_short_backtrace
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/backtrace.rs:171:18
   6: rust_begin_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:661:5
   7: core::panicking::panic_fmt
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panicking.rs:74:14
   8: core::result::unwrap_failed
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/result.rs:1679:5
   9: core::result::Result<T,E>::unwrap
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/result.rs:1102:23
      databend_query::pipelines::processors::transforms::aggregator::serde::transform_spill_reader::TransformSpillReader<Method,V>::deserialize
             at /workspace/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs:313:26
  10: <databend_query::pipelines::processors::transforms::aggregator::serde::transform_spill_reader::TransformSpillReader<Method,V> as databend_common_pipeline_core::processors::processor::Processor>::process
             at /workspace/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs:158:47
  11: databend_common_pipeline_core::processors::processor::ProcessorPtr::process
             at /workspace/src/query/pipeline/core/src/processors/processor.rs:169:9
  12: databend_query::pipelines::executor::executor_worker_context::ExecutorWorkerContext::execute_sync_task
             at /workspace/src/query/service/src/pipelines/executor/executor_worker_context.rs:169:9
      databend_query::pipelines::executor::executor_worker_context::ExecutorWorkerContext::execute_task
             at /workspace/src/query/service/src/pipelines/executor/executor_worker_context.rs:132:52
  13: databend_query::pipelines::executor::query_pipeline_executor::QueryPipelineExecutor::execute_single_thread
             at /workspace/src/query/service/src/pipelines/executor/query_pipeline_executor.rs:406:35
  14: databend_query::pipelines::executor::query_pipeline_executor::QueryPipelineExecutor::execute_threads::{{closure}}::{{closure}}
             at /workspace/src/query/service/src/pipelines/executor/query_pipeline_executor.rs:378:50
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
  15: databend_common_base::runtime::catch_unwind::catch_unwind
             at /workspace/src/common/base/src/runtime/catch_unwind.rs:47:11
  16: databend_query::pipelines::executor::query_pipeline_executor::QueryPipelineExecutor::execute_threads::{{closure}}
             at /workspace/src/query/service/src/pipelines/executor/query_pipeline_executor.rs:378:34
      databend_common_base::runtime::runtime_tracker::ThreadTracker::tracking_function::{{closure}}::{{closure}}
             at /workspace/src/common/base/src/runtime/runtime_tracker.rs:208:17
      databend_common_base::runtime::thread::Thread::named_spawn::{{closure}}
             at /workspace/src/common/base/src/runtime/thread.rs:78:21
      std::sys::backtrace::__rust_begin_short_backtrace
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/backtrace.rs:155:18
  17: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/mod.rs:542:17
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
      std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/mod.rs:541:30
      core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/ops/function.rs:250:5
  18: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2064:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2064:9
      std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/pal/unix/thread.rs:108:17
  19: start_thread
             at ./nptl/pthread_create.c:442:8
  20: __GI___clone3
             at ./misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
 panic.file=src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs panic.line=313 panic.column=91
0f480d68-1143-4ea5-80f1-834f3e51eeb9 2024-10-23T14:55:42.373898+08:00  INFO databend_enterprise_query::storages::fuse::operations::vacuum_temporary_files: vacuum_temporary_files.rs:122 vacuum removed 41 temp files in "_query_spill/rong/0f480d68-1143-4ea5-80f1-834f3e51eeb9/"(elapsed: 0 seconds), batch size: 2925523264 bytes
0f480d68-1143-4ea5-80f1-834f3e51eeb9 2024-10-23T14:55:42.373935+08:00  INFO databend_enterprise_query::storages::fuse::operations::vacuum_temporary_files: vacuum_temporary_files.rs:131 Total progress: 41 files removed, total cleaned size: 2925523264 bytes, total batch size: 2925523264 bytes
0f480d68-1143-4ea5-80f1-834f3e51eeb9 2024-10-23T14:55:42.373937+08:00  INFO databend_enterprise_query::storages::fuse::operations::vacuum_temporary_files: vacuum_temporary_files.rs:147 vacuum finished, total cleaned 41 files, total cleaned size: 2925523264 bytes, total elapsed: 0 seconds
0f480d68-1143-4ea5-80f1-834f3e51eeb9 2024-10-23T14:55:42.374332+08:00  INFO databend_query::interpreters::common::query_log: query_log.rs:72 query: 0f480d68-1143-4ea5-80f1-834f3e51eeb9 becomes Error
0f480d68-1143-4ea5-80f1-834f3e51eeb9 2024-10-23T14:55:42.374360+08:00  INFO databend_common_pipeline_core::finished_chain: finished_chain.rs:217 Executor apply finished callback state:
├──❌:17.291µs - src/query/service/src/interpreters/interpreter_select.rs:146:14
├──❌:0ns - (interrupt) src/query/service/src/pipelines/executor/pipeline_pulling_executor.rs:133:18
├──❌:17.03944ms - (always) src/query/service/src/interpreters/interpreter.rs:130:14

How to Reproduce?

TPCH-300 Q15 with 32G memory

Are you willing to submit PR?

• Yes I am willing to submit a PR!

[sundy-li]

Please upgrade to the latest version.

Fixed by #16657