ClickHouse/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h

55 lines
1.4 KiB
C++
Raw Normal View History

#pragma once
2021-07-24 11:38:42 +00:00
2021-10-27 23:10:39 +00:00
#include "config_formats.h"
2019-08-21 14:19:47 +00:00
2020-04-28 19:52:22 +00:00
#if USE_ARROW || USE_ORC || USE_PARQUET
2019-08-21 14:19:47 +00:00
#include <DataTypes/IDataType.h>
#include <Core/ColumnWithTypeAndName.h>
2021-08-09 14:30:14 +00:00
#include <Core/Block.h>
2019-08-21 14:19:47 +00:00
#include <arrow/table.h>
2021-07-24 11:38:42 +00:00
2019-08-21 14:19:47 +00:00
namespace DB
{
2021-07-24 11:38:42 +00:00
class Block;
class Chunk;
2021-06-07 15:15:58 +00:00
class ArrowColumnToCHColumn
{
public:
2021-10-31 19:53:24 +00:00
using NameToColumnPtr = std::unordered_map<std::string, std::shared_ptr<arrow::ChunkedArray>>;
2021-12-02 08:14:25 +00:00
ArrowColumnToCHColumn(
const Block & header_,
const std::string & format_name_,
bool import_nested_,
bool allow_missing_columns_);
2021-12-02 08:14:25 +00:00
void arrowTableToCHChunk(Chunk & res, std::shared_ptr<arrow::Table> & table);
2019-08-21 14:19:47 +00:00
2021-12-02 08:14:25 +00:00
void arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & name_to_column_ptr);
/// Get missing columns that exists in header but not in arrow::Schema
std::vector<size_t> getMissingColumns(const arrow::Schema & schema) const;
2021-10-31 19:53:24 +00:00
static Block arrowSchemaToCHHeader(const arrow::Schema & schema, const std::string & format_name);
2021-06-07 15:15:58 +00:00
private:
const Block & header;
2021-06-07 15:15:58 +00:00
const std::string format_name;
bool import_nested;
2021-12-02 08:14:25 +00:00
/// If false, throw exception if some columns in header not exists in arrow table.
bool allow_missing_columns;
2021-08-05 15:09:48 +00:00
2021-06-07 15:15:58 +00:00
/// Map {column name : dictionary column}.
/// To avoid converting dictionary from Arrow Dictionary
/// to LowCardinality every chunk we save it and reuse.
std::unordered_map<std::string, std::shared_ptr<ColumnWithTypeAndName>> dictionary_values;
2021-06-07 15:15:58 +00:00
};
2021-07-24 11:38:42 +00:00
2019-08-21 14:19:47 +00:00
}
2021-07-24 11:38:42 +00:00
2019-08-21 14:19:47 +00:00
#endif