orc_metadata.hpp

转到此文件的文档。

/*
 * Copyright (c) 2019-2025, NVIDIA CORPORATION.
 *
 * 根据 Apache 许可证 2.0 版（“许可证”）获得许可；
 * 除非遵守许可证，否则您不得使用此文件。
 * 您可以获取许可证的副本，地址如下：
 *
 * https://apache.ac.cn/licenses/LICENSE-2.0
 *
 * 除非适用法律要求或书面同意，软件
 * 根据许可证分发时按“原样”分发，
 * 不附带任何明示或暗示的保证或条件。
 * 请参阅许可证以了解特定语言的管理权限和
 * 限制。
 */

#pragma once

#include <cudf/io/orc_types.hpp>
#include <cudf/io/types.hpp>
#include <cudf/utilities/export.hpp>

#include <optional>
#include <variant>
#include <vector>

namespace CUDF_EXPORT cudf {
namespace io {
struct raw_orc_statistics {
 std::vector<std::string> column_names;
 std::vector<std::string> file_stats;
 std::vector<std::vector<std::string>> stripes_stats;
};

raw_orc_statistics read_raw_orc_statistics(
 source_info const& src_info, rmm::cuda_stream_view stream = cudf::get_default_stream());

using no_statistics = std::monostate;

template <typename T>
struct minmax_statistics {
 std::optional<T> minimum;
 std::optional<T> maximum;
};

template <typename T>
struct sum_statistics {
 std::optional<T> sum;
};

struct integer_statistics : minmax_statistics<int64_t>, sum_statistics<int64_t> {};

struct double_statistics : minmax_statistics<double>, sum_statistics<double> {};

struct string_statistics : minmax_statistics<std::string>, sum_statistics<int64_t> {};

struct bucket_statistics {
 std::vector<uint64_t> count;
};

struct decimal_statistics : minmax_statistics<std::string>, sum_statistics<std::string> {};

using date_statistics = minmax_statistics<int32_t>;

using binary_statistics = sum_statistics<int64_t>;

struct timestamp_statistics : minmax_statistics<int64_t> {
 std::optional<int64_t> minimum_utc;
 std::optional<int64_t> maximum_utc;
 std::optional<uint32_t> minimum_nanos;
 std::optional<uint32_t> maximum_nanos;
};

using statistics_type = std::variant<no_statistics,
 integer_statistics,
 double_statistics,
 string_statistics,
 bucket_statistics,
 decimal_statistics,
 date_statistics,
 binary_statistics,
 timestamp_statistics>;

namespace orc::detail {
// 前向声明 protobuf_reader 使用的类型。`cudf::io::column_statistics` 对象，
// 从 `read_parsed_orc_statistics` 返回的对象，是从
// `protobuf_reader` 初始化的 `cudf::io::orc::detail::column_statistics` 对象构造的。
struct column_statistics;
} // namespace orc::detail

struct column_statistics {
 std::optional<uint64_t> number_of_values;
 std::optional<bool> has_null;
 statistics_type type_specific_stats;

 column_statistics(orc::detail::column_statistics&& detail_statistics);
};

struct parsed_orc_statistics {
 std::vector<std::string> column_names;
 std::vector<column_statistics> file_stats;
 std::vector<std::vector<column_statistics>> stripes_stats;
};

parsed_orc_statistics read_parsed_orc_statistics(
 source_info const& src_info, rmm::cuda_stream_view stream = cudf::get_default_stream());

struct orc_column_schema {
 public
 orc_column_schema(std::string_view name,
 orc::TypeKind type,
 std::vector<orc_column_schema> children)
 : _name{name}, _type_kind{type}, _children{std::move(children)}
 {
 }

 [[nodiscard]] auto name() const { return _name; }

 [[nodiscard]] auto type_kind() const { return _type_kind; }

 [[nodiscard]] auto const& children() const& { return _children; }

 [[nodiscard]] auto children() && { return std::move(_children); }

 [[nodiscard]] auto const& child(int idx) const& { return children().at(idx); }

 [[nodiscard]] auto child(int idx) && { return std::move(children().at(idx)); }

 [[nodiscard]] auto num_children() const { return children().size(); }

 private
 std::string _name;
 orc::TypeKind _type_kind;
 std::vector<orc_column_schema> _children;
};

struct orc_schema {
 public
 orc_schema(orc_column_schema root_column_schema) : _root{std::move(root_column_schema)} {}

 [[nodiscard]] auto const& root() const& { return _root; }

 [[nodiscard]] auto root() && { return std::move(_root); }

 private
 orc_column_schema _root;
};

class orc_metadata {
 public
 orc_metadata(orc_schema schema, uint64_t num_rows, size_type num_stripes)
 : _schema{std::move(schema)}, _num_rows{num_rows}, _num_stripes{num_stripes}
 {
 }

 [[nodiscard]] auto const& schema() const { return _schema; }


 [[nodiscard]] auto num_rows() const { return _num_rows; }

 [[nodiscard]] auto num_stripes() const { return _num_stripes; }

 private
 orc_schema _schema;
 uint64_t _num_rows;
 size_type _num_stripes;
};

orc_metadata read_orc_metadata(source_info const& src_info,
 rmm::cuda_stream_view stream = cudf::get_default_stream());
 // 组结束
} // namespace io
} // namespace CUDF_EXPORT cudf