Skip to content

Commit

Permalink
apacheGH-41909: [C++] Add arrow::ArrayStatistics
Browse files Browse the repository at this point in the history
See apacheGH-42133 how to use this for Apache Parquet statistics.
  • Loading branch information
kou committed Jul 16, 2024
1 parent 21238a7 commit f9136fb
Show file tree
Hide file tree
Showing 3 changed files with 180 additions and 0 deletions.
1 change: 1 addition & 0 deletions cpp/src/arrow/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1168,6 +1168,7 @@ add_arrow_test(array_test
array/array_struct_test.cc
array/array_union_test.cc
array/array_view_test.cc
array/statistics_test.cc
PRECOMPILED_HEADERS
"$<$<COMPILE_LANGUAGE:CXX>:arrow/testing/pch.h>")

Expand Down
76 changes: 76 additions & 0 deletions cpp/src/arrow/array/statistics.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#pragma once

#include <optional>
#include <variant>

#include "arrow/util/visibility.h"

namespace arrow {

/// \brief Statistics for an Array
///
/// Apache Arrow format doesn't have statistics but data source such
/// as Apache Parquet may have statistics. Statistics associate with
/// data source can be read unified API via this class.
struct ARROW_EXPORT ArrayStatistics {
public:
using ElementBufferType = std::variant<bool, int8_t, uint8_t, int16_t, uint16_t,
int32_t, uint32_t, int64_t, uint64_t>;

ArrayStatistics() = default;
~ArrayStatistics() = default;

/// \brief The number of null values, may not be set
std::optional<int64_t> null_count = std::nullopt;

/// \brief The number of distinct values, may not be set
std::optional<int64_t> distinct_count = std::nullopt;

/// \brief The minimum value buffer, may not be set
std::optional<ElementBufferType> min_buffer = std::nullopt;

/// \brief Whether the minimum value is exact or not, may not be set
std::optional<bool> is_min_exact = std::nullopt;

/// \brief The maximum value buffer, may not be set
std::optional<ElementBufferType> max_buffer = std::nullopt;

/// \brief Whether the maximum value is exact or not, may not be set
std::optional<bool> is_max_exact = std::nullopt;

/// \brief Check two statistics for equality
bool Equals(const ArrayStatistics& other) const {
return null_count == other.null_count && distinct_count == other.distinct_count &&
min_buffer == other.min_buffer && is_min_exact == other.is_min_exact &&
max_buffer == other.max_buffer && is_max_exact == other.is_max_exact;
}

/// \brief Check two statistics for equality
bool operator==(const ArrayStatistics& other) const {
return Equals(other);
}

/// \brief Check two statistics for not equality
bool operator!=(const ArrayStatistics& other) const {
return !Equals(other);
}
};

} // namespace arrow
103 changes: 103 additions & 0 deletions cpp/src/arrow/array/statistics_test.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include <gtest/gtest.h>

#include "arrow/array/statistics.h"

namespace arrow {

TEST(ArrayStatisticsTest, TestNullCount) {
ArrayStatistics statistics;
ASSERT_FALSE(statistics.null_count.has_value());
statistics.null_count = 29;
ASSERT_TRUE(statistics.null_count.has_value());
ASSERT_EQ(29, statistics.null_count.value());
}

TEST(ArrayStatisticsTest, TestDistinctCount) {
ArrayStatistics statistics;
ASSERT_FALSE(statistics.distinct_count.has_value());
statistics.distinct_count = 29;
ASSERT_TRUE(statistics.distinct_count.has_value());
ASSERT_EQ(29, statistics.distinct_count.value());
}

TEST(ArrayStatisticsTest, TestMin) {
ArrayStatistics statistics;
ASSERT_FALSE(statistics.min_buffer.has_value());
ASSERT_FALSE(statistics.is_min_exact.has_value());
statistics.min_buffer = static_cast<int32_t>(29);
statistics.is_min_exact = true;
ASSERT_TRUE(statistics.min_buffer.has_value());
ASSERT_TRUE(std::holds_alternative<int32_t>(statistics.min_buffer.value()));
ASSERT_EQ(29, std::get<int32_t>(statistics.min_buffer.value()));
ASSERT_TRUE(statistics.is_min_exact.has_value());
ASSERT_TRUE(statistics.is_min_exact.value());
}

TEST(ArrayStatisticsTest, TestMax) {
ArrayStatistics statistics;
ASSERT_FALSE(statistics.max_buffer.has_value());
ASSERT_FALSE(statistics.is_max_exact.has_value());
statistics.max_buffer = static_cast<int32_t>(29);
statistics.is_max_exact = false;
ASSERT_TRUE(statistics.max_buffer.has_value());
ASSERT_TRUE(std::holds_alternative<int32_t>(statistics.max_buffer.value()));
ASSERT_EQ(29, std::get<int32_t>(statistics.max_buffer.value()));
ASSERT_TRUE(statistics.is_max_exact.has_value());
ASSERT_FALSE(statistics.is_max_exact.value());
}

TEST(ArrayStatisticsTest, TestEquality) {
ArrayStatistics statistics1;
ArrayStatistics statistics2;

ASSERT_EQ(statistics1, statistics2);

statistics1.null_count = 29;
ASSERT_NE(statistics1, statistics2);
statistics2.null_count = 29;
ASSERT_EQ(statistics1, statistics2);

statistics1.distinct_count = 2929;
ASSERT_NE(statistics1, statistics2);
statistics2.distinct_count = 2929;
ASSERT_EQ(statistics1, statistics2);

statistics1.min_buffer = static_cast<uint8_t>(255);
ASSERT_NE(statistics1, statistics2);
statistics2.min_buffer = static_cast<uint8_t>(255);
ASSERT_EQ(statistics1, statistics2);

statistics1.is_min_exact = false;
ASSERT_NE(statistics1, statistics2);
statistics2.is_min_exact = false;
ASSERT_EQ(statistics1, statistics2);

statistics1.max_buffer = static_cast<int64_t>(-255);
ASSERT_NE(statistics1, statistics2);
statistics2.max_buffer = static_cast<int64_t>(-255);
ASSERT_EQ(statistics1, statistics2);

statistics1.is_max_exact = true;
ASSERT_NE(statistics1, statistics2);
statistics2.is_max_exact = true;
ASSERT_EQ(statistics1, statistics2);
}

} // namespace arrow

0 comments on commit f9136fb

Please sign in to comment.