mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-27 01:51:59 +00:00
Miscellaneous #2913
This commit is contained in:
Alexey Milovidov
parent
6043ea02de
commit
761422da22
@ -22,9 +22,8 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* BlockInputStream implementation for external dictionaries
|
||||
* read() returns single block consisting of the in-memory contents of the dictionaries
|
||||
/* BlockInputStream implementation for external dictionaries
|
||||
* read() returns blocks consisting of the in-memory contents of the dictionaries
|
||||
*/
|
||||
template <typename DictionaryType, typename Key>
|
||||
class DictionaryBlockInputStream : public DictionaryBlockInputStreamBase
|
||||
@ -34,11 +33,13 @@ public:
|
||||
|
||||
DictionaryBlockInputStream(std::shared_ptr<const IDictionaryBase> dictionary, size_t max_block_size,
|
||||
PaddedPODArray<Key> && ids, const Names & column_names);
|
||||
|
||||
DictionaryBlockInputStream(std::shared_ptr<const IDictionaryBase> dictionary, size_t max_block_size,
|
||||
const std::vector<StringRef> & keys, const Names & column_names);
|
||||
|
||||
using GetColumnsFunction =
|
||||
std::function<ColumnsWithTypeAndName(const Columns &, const std::vector<DictionaryAttribute> & attributes)>;
|
||||
|
||||
// Used to separate key columns format for storage and view.
|
||||
// Calls get_key_columns_function to get key column for dictionary get fuction call
|
||||
// and get_view_columns_function to get key representation.
|
||||
@ -60,16 +61,15 @@ private:
|
||||
// pointer types to getXXX functions
|
||||
// for single key dictionaries
|
||||
template <typename Type>
|
||||
using DictionaryGetter = void (DictionaryType::*)(
|
||||
const std::string &, const PaddedPODArray<Key> &, PaddedPODArray<Type> &) const;
|
||||
using DictionaryStringGetter = void (DictionaryType::*)(
|
||||
const std::string &, const PaddedPODArray<Key> &, ColumnString *) const;
|
||||
using DictionaryGetter = void (DictionaryType::*)(const std::string &, const PaddedPODArray<Key> &, PaddedPODArray<Type> &) const;
|
||||
|
||||
using DictionaryStringGetter = void (DictionaryType::*)(const std::string &, const PaddedPODArray<Key> &, ColumnString *) const;
|
||||
|
||||
// for complex complex key dictionaries
|
||||
template <typename Type>
|
||||
using GetterByKey = void (DictionaryType::*)(
|
||||
const std::string &, const Columns &, const DataTypes &, PaddedPODArray<Type> & out) const;
|
||||
using StringGetterByKey = void (DictionaryType::*)(
|
||||
const std::string &, const Columns &, const DataTypes &, ColumnString * out) const;
|
||||
using GetterByKey = void (DictionaryType::*)(const std::string &, const Columns &, const DataTypes &, PaddedPODArray<Type> & out) const;
|
||||
|
||||
using StringGetterByKey = void (DictionaryType::*)(const std::string &, const Columns &, const DataTypes &, ColumnString * out) const;
|
||||
|
||||
// call getXXX
|
||||
// for single key dictionaries
|
||||
@ -77,15 +77,18 @@ private:
|
||||
void callGetter(DictionaryGetter<Type> getter, const PaddedPODArray<Key> & ids,
|
||||
const Columns & keys, const DataTypes & data_types,
|
||||
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const;
|
||||
|
||||
template <typename Container>
|
||||
void callGetter(DictionaryStringGetter getter, const PaddedPODArray<Key> & ids,
|
||||
const Columns & keys, const DataTypes & data_types,
|
||||
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const;
|
||||
|
||||
// for complex complex key dictionaries
|
||||
template <typename Type, typename Container>
|
||||
void callGetter(GetterByKey<Type> getter, const PaddedPODArray<Key> & ids,
|
||||
const Columns & keys, const DataTypes & data_types,
|
||||
Container & container, const DictionaryAttribute & attribute, const DictionaryType & dictionary) const;
|
||||
|
||||
template <typename Container>
|
||||
void callGetter(StringGetterByKey getter, const PaddedPODArray<Key> & ids,
|
||||
const Columns & keys, const DataTypes & data_types,
|
||||
@ -114,10 +117,13 @@ private:
|
||||
PaddedPODArray<Key> ids;
|
||||
ColumnsWithTypeAndName key_columns;
|
||||
Poco::Logger * logger;
|
||||
Block (DictionaryBlockInputStream<DictionaryType, Key>::*fillBlockFunction)(
|
||||
|
||||
using FillBlockFunction = Block (DictionaryBlockInputStream<DictionaryType, Key>::*)(
|
||||
const PaddedPODArray<Key> & ids, const Columns & keys,
|
||||
const DataTypes & types, ColumnsWithTypeAndName && view) const;
|
||||
|
||||
FillBlockFunction fill_block_function;
|
||||
|
||||
Columns data_columns;
|
||||
GetColumnsFunction get_key_columns_function;
|
||||
GetColumnsFunction get_view_columns_function;
|
||||
@ -132,6 +138,7 @@ private:
|
||||
DictionaryKeyType key_type;
|
||||
};
|
||||
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
DictionaryBlockInputStream<DictionaryType, Key>::DictionaryBlockInputStream(
|
||||
std::shared_ptr<const IDictionaryBase> dictionary, size_t max_block_size,
|
||||
@ -140,7 +147,7 @@ DictionaryBlockInputStream<DictionaryType, Key>::DictionaryBlockInputStream(
|
||||
dictionary(std::static_pointer_cast<const DictionaryType>(dictionary)),
|
||||
column_names(column_names), ids(std::move(ids)),
|
||||
logger(&Poco::Logger::get("DictionaryBlockInputStream")),
|
||||
fillBlockFunction(&DictionaryBlockInputStream<DictionaryType, Key>::fillBlock<DictionaryGetter, DictionaryStringGetter>),
|
||||
fill_block_function(&DictionaryBlockInputStream<DictionaryType, Key>::fillBlock<DictionaryGetter, DictionaryStringGetter>),
|
||||
key_type(DictionaryKeyType::Id)
|
||||
{
|
||||
}
|
||||
@ -152,7 +159,7 @@ DictionaryBlockInputStream<DictionaryType, Key>::DictionaryBlockInputStream(
|
||||
: DictionaryBlockInputStreamBase(keys.size(), max_block_size),
|
||||
dictionary(std::static_pointer_cast<const DictionaryType>(dictionary)), column_names(column_names),
|
||||
logger(&Poco::Logger::get("DictionaryBlockInputStream")),
|
||||
fillBlockFunction(&DictionaryBlockInputStream<DictionaryType, Key>::fillBlock<GetterByKey, StringGetterByKey>),
|
||||
fill_block_function(&DictionaryBlockInputStream<DictionaryType, Key>::fillBlock<GetterByKey, StringGetterByKey>),
|
||||
key_type(DictionaryKeyType::ComplexKey)
|
||||
{
|
||||
const DictionaryStructure & dictionaty_structure = dictionary->getStructure();
|
||||
@ -168,13 +175,14 @@ DictionaryBlockInputStream<DictionaryType, Key>::DictionaryBlockInputStream(
|
||||
: DictionaryBlockInputStreamBase(data_columns.front()->size(), max_block_size),
|
||||
dictionary(std::static_pointer_cast<const DictionaryType>(dictionary)), column_names(column_names),
|
||||
logger(&Poco::Logger::get("DictionaryBlockInputStream")),
|
||||
fillBlockFunction(&DictionaryBlockInputStream<DictionaryType, Key>::fillBlock<GetterByKey, StringGetterByKey>),
|
||||
fill_block_function(&DictionaryBlockInputStream<DictionaryType, Key>::fillBlock<GetterByKey, StringGetterByKey>),
|
||||
data_columns(data_columns),
|
||||
get_key_columns_function(get_key_columns_function), get_view_columns_function(get_view_columns_function),
|
||||
key_type(DictionaryKeyType::Callback)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
Block DictionaryBlockInputStream<DictionaryType, Key>::getBlock(size_t start, size_t length) const
|
||||
{
|
||||
@ -191,13 +199,15 @@ Block DictionaryBlockInputStream<DictionaryType, Key>::getBlock(size_t start, si
|
||||
columns.emplace_back(column);
|
||||
view_columns.emplace_back(column, key_column.type, key_column.name);
|
||||
}
|
||||
return (this->*fillBlockFunction)({}, columns, {}, std::move(view_columns));
|
||||
return (this->*fill_block_function)({}, columns, {}, std::move(view_columns));
|
||||
}
|
||||
|
||||
case DictionaryKeyType::Id:
|
||||
{
|
||||
PaddedPODArray<Key> block_ids(ids.begin() + start, ids.begin() + start + length);
|
||||
return (this->*fillBlockFunction)(block_ids, {}, {}, {});
|
||||
return (this->*fill_block_function)(block_ids, {}, {}, {});
|
||||
}
|
||||
|
||||
case DictionaryKeyType::Callback:
|
||||
{
|
||||
Columns columns;
|
||||
@ -215,12 +225,14 @@ Block DictionaryBlockInputStream<DictionaryType, Key>::getBlock(size_t start, si
|
||||
columns.push_back(key_column.column);
|
||||
types.push_back(key_column.type);
|
||||
}
|
||||
return (this->*fillBlockFunction)({}, columns, types, std::move(view_with_type_and_name));
|
||||
return (this->*fill_block_function)({}, columns, types, std::move(view_with_type_and_name));
|
||||
}
|
||||
}
|
||||
|
||||
throw Exception("Unexpected DictionaryKeyType.", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
template <typename Type, typename Container>
|
||||
void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
|
||||
@ -261,6 +273,7 @@ void DictionaryBlockInputStream<DictionaryType, Key>::callGetter(
|
||||
(dict.*getter)(attribute.name, keys, data_types, container);
|
||||
}
|
||||
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
template <template <typename> class Getter, typename StringGetter>
|
||||
Block DictionaryBlockInputStream<DictionaryType, Key>::fillBlock(
|
||||
@ -344,6 +357,7 @@ Block DictionaryBlockInputStream<DictionaryType, Key>::fillBlock(
|
||||
return Block(block_columns);
|
||||
}
|
||||
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
template <typename AttributeType, typename Getter>
|
||||
ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromAttribute(
|
||||
@ -359,6 +373,7 @@ ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromAttribut
|
||||
return std::move(column_vector);
|
||||
}
|
||||
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
template <typename Getter>
|
||||
ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromStringAttribute(
|
||||
@ -372,6 +387,7 @@ ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromStringAt
|
||||
return std::move(column_string);
|
||||
}
|
||||
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromIds(const PaddedPODArray<Key> & ids_to_fill) const
|
||||
{
|
||||
@ -382,6 +398,7 @@ ColumnPtr DictionaryBlockInputStream<DictionaryType, Key>::getColumnFromIds(cons
|
||||
return std::move(column_vector);
|
||||
}
|
||||
|
||||
|
||||
template <typename DictionaryType, typename Key>
|
||||
void DictionaryBlockInputStream<DictionaryType, Key>::fillKeyColumns(
|
||||
const std::vector<StringRef> & keys, size_t start, size_t size,
|
||||
|
||||
@ -4,7 +4,7 @@ namespace DB
|
||||
{
|
||||
|
||||
DictionaryBlockInputStreamBase::DictionaryBlockInputStreamBase(size_t rows_count, size_t max_block_size)
|
||||
: rows_count(rows_count), max_block_size(max_block_size), next_row(0)
|
||||
: rows_count(rows_count), max_block_size(max_block_size)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#pragma once
|
||||
|
||||
#include <DataStreams/IProfilingBlockInputStream.h>
|
||||
|
||||
namespace DB
|
||||
@ -7,8 +8,6 @@ namespace DB
|
||||
class DictionaryBlockInputStreamBase : public IProfilingBlockInputStream
|
||||
{
|
||||
protected:
|
||||
//Block block;
|
||||
|
||||
DictionaryBlockInputStreamBase(size_t rows_count, size_t max_block_size);
|
||||
|
||||
virtual Block getBlock(size_t start, size_t length) const = 0;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user