thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 15:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #25119 from kitaisreal/dictionaries-support-array-type

Browse Source 
Dictionaries support array type
This commit is contained in:
Maksim Kita 2021-06-10 23:34:51 +03:00 committed by GitHub
commit 4dd8470833
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 617 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

204
src/Dictionaries/CacheDictionaryStorage.h
View File

@ -55,7 +55,7 @@ public:
cells.resize_fill(cells_size); cells.resize_fill(cells_size);
size_overlap_mask = cells_size - 1; size_overlap_mask = cells_size - 1;
setup(dictionary_structure); createAttributes(dictionary_structure);
} }
bool returnsFetchedColumnsInOrderOfRequestedKeys() const override { return true; } bool returnsFetchedColumnsInOrderOfRequestedKeys() const override { return true; }
@ -226,23 +226,17 @@ private:
auto & attribute = attributes[attribute_index]; auto & attribute = attributes[attribute_index];
const auto & default_value_provider = fetch_request.defaultValueProviderAtIndex(attribute_index); const auto & default_value_provider = fetch_request.defaultValueProviderAtIndex(attribute_index);
size_t fetched_keys_size = fetched_keys.size();
auto & fetched_column = *result.fetched_columns[attribute_index]; auto & fetched_column = *result.fetched_columns[attribute_index];
fetched_column.reserve(fetched_keys_size); fetched_column.reserve(fetched_columns_index);
if (unlikely(attribute.is_complex_type)) if (unlikely(attribute.is_nullable))
{ {
auto & container = std::get<std::vector<Field>>(attribute.attribute_container); getItemsForFetchedKeys<Field>(
attribute,
for (size_t fetched_key_index = 0; fetched_key_index < fetched_columns_index; ++fetched_key_index) fetched_columns_index,
{ fetched_keys,
auto fetched_key = fetched_keys[fetched_key_index]; [&](Field & value) { fetched_column.insert(value); },
default_value_provider);
if (unlikely(fetched_key.is_default))
fetched_column.insert(default_value_provider.getDefaultValue(fetched_key_index));
else
fetched_column.insert(container[fetched_key.element_index]);
}
} }
else else
{ {
@ -250,46 +244,40 @@ private:
{ {
using Type = std::decay_t<decltype(dictionary_attribute_type)>; using Type = std::decay_t<decltype(dictionary_attribute_type)>;
using AttributeType = typename Type::AttributeType; using AttributeType = typename Type::AttributeType;
using ColumnProvider = DictionaryAttributeColumnProvider<AttributeType>;
using ColumnType = typename ColumnProvider::ColumnType;
using ValueType = DictionaryValueType<AttributeType>; using ValueType = DictionaryValueType<AttributeType>;
using ColumnType =
std::conditional_t<std::is_same_v<AttributeType, String>, ColumnString,
std::conditional_t<IsDecimalNumber<AttributeType>, ColumnDecimal<ValueType>,
ColumnVector<AttributeType>>>;
auto & container = std::get<PaddedPODArray<ValueType>>(attribute.attribute_container);
ColumnType & column_typed = static_cast<ColumnType &>(fetched_column); ColumnType & column_typed = static_cast<ColumnType &>(fetched_column);
if constexpr (std::is_same_v<ColumnType, ColumnString>) if constexpr (std::is_same_v<ValueType, Array>)
{ {
for (size_t fetched_key_index = 0; fetched_key_index < fetched_columns_index; ++fetched_key_index) getItemsForFetchedKeys<ValueType>(
{ attribute,
auto fetched_key = fetched_keys[fetched_key_index]; fetched_columns_index,
fetched_keys,
if (unlikely(fetched_key.is_default)) [&](Array & value) { fetched_column.insert(value); },
column_typed.insert(default_value_provider.getDefaultValue(fetched_key_index)); default_value_provider);
else }
{ else if constexpr (std::is_same_v<ValueType, StringRef>)
auto item = container[fetched_key.element_index]; {
column_typed.insertData(item.data, item.size); getItemsForFetchedKeys<ValueType>(
} attribute,
} fetched_columns_index,
fetched_keys,
[&](StringRef value) { fetched_column.insertData(value.data, value.size); },
default_value_provider);
} }
else else
{ {
auto & data = column_typed.getData(); auto & data = column_typed.getData();
for (size_t fetched_key_index = 0; fetched_key_index < fetched_columns_index; ++fetched_key_index) getItemsForFetchedKeys<ValueType>(
{ attribute,
auto fetched_key = fetched_keys[fetched_key_index]; fetched_columns_index,
fetched_keys,
if (unlikely(fetched_key.is_default)) [&](auto value) { data.push_back(value); },
column_typed.insert(default_value_provider.getDefaultValue(fetched_key_index)); default_value_provider);
else
{
auto item = container[fetched_key.element_index];
data.push_back(item);
}
}
} }
}; };
@ -339,7 +327,9 @@ private:
column->get(key_index, column_value); column->get(key_index, column_value);
if constexpr (std::is_same_v<ElementType, Field>) if constexpr (std::is_same_v<ElementType, Field>)
{
container.back() = column_value; container.back() = column_value;
}
else if constexpr (std::is_same_v<ElementType, StringRef>) else if constexpr (std::is_same_v<ElementType, StringRef>)
{ {
const String & string_value = column_value.get<String>(); const String & string_value = column_value.get<String>();
@ -348,7 +338,9 @@ private:
container.back() = inserted_value; container.back() = inserted_value;
} }
else else
{
container.back() = column_value.get<NearestFieldType<ElementType>>(); container.back() = column_value.get<NearestFieldType<ElementType>>();
}
}); });
} }
@ -382,7 +374,9 @@ private:
column->get(key_index, column_value); column->get(key_index, column_value);
if constexpr (std::is_same_v<ElementType, Field>) if constexpr (std::is_same_v<ElementType, Field>)
{
container[index_to_use] = column_value; container[index_to_use] = column_value;
}
else if constexpr (std::is_same_v<ElementType, StringRef>) else if constexpr (std::is_same_v<ElementType, StringRef>)
{ {
const String & string_value = column_value.get<String>(); const String & string_value = column_value.get<String>();
@ -398,7 +392,9 @@ private:
container[index_to_use] = inserted_value; container[index_to_use] = inserted_value;
} }
else else
{
container[index_to_use] = column_value.get<NearestFieldType<ElementType>>(); container[index_to_use] = column_value.get<NearestFieldType<ElementType>>();
}
}); });
} }
} }
@ -504,9 +500,9 @@ private:
auto & attribute = attributes[attribute_index]; auto & attribute = attributes[attribute_index];
auto & attribute_type = attribute.type; auto & attribute_type = attribute.type;
if (unlikely(attribute.is_complex_type)) if (unlikely(attribute.is_nullable))
{ {
auto & container = std::get<std::vector<Field>>(attribute.attribute_container); auto & container = std::get<ContainerType<Field>>(attribute.attribute_container);
std::forward<GetContainerFunc>(func)(container); std::forward<GetContainerFunc>(func)(container);
} }
else else
@ -517,7 +513,7 @@ private:
using AttributeType = typename Type::AttributeType; using AttributeType = typename Type::AttributeType;
using ValueType = DictionaryValueType<AttributeType>; using ValueType = DictionaryValueType<AttributeType>;
auto & container = std::get<PaddedPODArray<ValueType>>(attribute.attribute_container); auto & container = std::get<ContainerType<ValueType>>(attribute.attribute_container);
std::forward<GetContainerFunc>(func)(container); std::forward<GetContainerFunc>(func)(container);
}; };
@ -541,7 +537,82 @@ private:
return updated_value; return updated_value;
} }
void setup(const DictionaryStructure & dictionary_structure) template<typename ValueType>
using ContainerType = std::conditional_t<
std::is_same_v<ValueType, Field> || std::is_same_v<ValueType, Array>,
std::vector<ValueType>,
PaddedPODArray<ValueType>>;
struct Attribute
{
AttributeUnderlyingType type;
bool is_nullable;
std::variant<
ContainerType<UInt8>,
ContainerType<UInt16>,
ContainerType<UInt32>,
ContainerType<UInt64>,
ContainerType<UInt128>,
ContainerType<UInt256>,
ContainerType<Int8>,
ContainerType<Int16>,
ContainerType<Int32>,
ContainerType<Int64>,
ContainerType<Int128>,
ContainerType<Int256>,
ContainerType<Decimal32>,
ContainerType<Decimal64>,
ContainerType<Decimal128>,
ContainerType<Decimal256>,
ContainerType<Float32>,
ContainerType<Float64>,
ContainerType<UUID>,
ContainerType<StringRef>,
ContainerType<Array>,
ContainerType<Field>> attribute_container;
};
template <typename ValueType, typename ValueSetter>
void getItemsForFetchedKeys(
Attribute & attribute,
size_t fetched_keys_size,
PaddedPODArray<FetchedKey> & fetched_keys,
ValueSetter && value_setter,
const DefaultValueProvider & default_value_provider)
{
auto & container = std::get<ContainerType<ValueType>>(attribute.attribute_container);
for (size_t fetched_key_index = 0; fetched_key_index < fetched_keys_size; ++fetched_key_index)
{
auto fetched_key = fetched_keys[fetched_key_index];
if (unlikely(fetched_key.is_default))
{
auto default_value = default_value_provider.getDefaultValue(fetched_key_index);
if constexpr (std::is_same_v<ValueType, Field>)
{
value_setter(default_value);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
{
auto & value = default_value.get<String>();
value_setter(value);
}
else
{
value_setter(default_value.get<ValueType>());
}
}
else
{
value_setter(container[fetched_key.element_index]);
}
}
}
void createAttributes(const DictionaryStructure & dictionary_structure)
{ {
/// For each dictionary attribute create storage attribute /// For each dictionary attribute create storage attribute
/// For simple attributes create PODArray, for complex vector of Fields /// For simple attributes create PODArray, for complex vector of Fields
@ -561,12 +632,12 @@ private:
attributes.emplace_back(); attributes.emplace_back();
auto & last_attribute = attributes.back(); auto & last_attribute = attributes.back();
last_attribute.type = attribute_type; last_attribute.type = attribute_type;
last_attribute.is_complex_type = dictionary_attribute.is_nullable || dictionary_attribute.is_array; last_attribute.is_nullable = dictionary_attribute.is_nullable;
if (dictionary_attribute.is_nullable) if (dictionary_attribute.is_nullable)
last_attribute.attribute_container = std::vector<Field>(); last_attribute.attribute_container = ContainerType<Field>();
else else
last_attribute.attribute_container = PaddedPODArray<ValueType>(); last_attribute.attribute_container = ContainerType<ValueType>();
}; };
callOnDictionaryAttributeType(attribute_type, type_call); callOnDictionaryAttributeType(attribute_type, type_call);
@ -583,35 +654,6 @@ private:
time_t deadline; time_t deadline;
}; };
struct Attribute
{
AttributeUnderlyingType type;
bool is_complex_type;
std::variant<
PaddedPODArray<UInt8>,
PaddedPODArray<UInt16>,
PaddedPODArray<UInt32>,
PaddedPODArray<UInt64>,
PaddedPODArray<UInt128>,
PaddedPODArray<UInt256>,
PaddedPODArray<Int8>,
PaddedPODArray<Int16>,
PaddedPODArray<Int32>,
PaddedPODArray<Int64>,
PaddedPODArray<Int128>,
PaddedPODArray<Int256>,
PaddedPODArray<Decimal32>,
PaddedPODArray<Decimal64>,
PaddedPODArray<Decimal128>,
PaddedPODArray<Decimal256>,
PaddedPODArray<Float32>,
PaddedPODArray<Float64>,
PaddedPODArray<UUID>,
PaddedPODArray<StringRef>,
std::vector<Field>> attribute_container;
};
CacheDictionaryStorageConfiguration configuration; CacheDictionaryStorageConfiguration configuration;
pcg64 rnd_engine; pcg64 rnd_engine;

42
src/Dictionaries/DictionaryHelpers.h
View File

@ -6,8 +6,11 @@
#include <Columns/ColumnDecimal.h> #include <Columns/ColumnDecimal.h>
#include <Columns/ColumnString.h> #include <Columns/ColumnString.h>
#include <Columns/ColumnVector.h> #include <Columns/ColumnVector.h>
#include <Columns/ColumnArray.h>
#include <DataStreams/IBlockInputStream.h> #include <DataStreams/IBlockInputStream.h>
#include <DataTypes/DataTypesDecimal.h> #include <DataTypes/DataTypesDecimal.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeNullable.h>
#include <Core/Block.h> #include <Core/Block.h>
#include <Dictionaries/IDictionary.h> #include <Dictionaries/IDictionary.h>
#include <Dictionaries/DictionaryStructure.h> #include <Dictionaries/DictionaryStructure.h>
@ -231,14 +234,27 @@ class DictionaryAttributeColumnProvider
{ {
public: public:
using ColumnType = using ColumnType =
std::conditional_t<std::is_same_v<DictionaryAttributeType, String>, ColumnString, std::conditional_t<std::is_same_v<DictionaryAttributeType, Array>, ColumnArray,
std::conditional_t<IsDecimalNumber<DictionaryAttributeType>, ColumnDecimal<DictionaryAttributeType>, std::conditional_t<std::is_same_v<DictionaryAttributeType, String>, ColumnString,
ColumnVector<DictionaryAttributeType>>>; std::conditional_t<IsDecimalNumber<DictionaryAttributeType>, ColumnDecimal<DictionaryAttributeType>,
ColumnVector<DictionaryAttributeType>>>>;
using ColumnPtr = typename ColumnType::MutablePtr; using ColumnPtr = typename ColumnType::MutablePtr;
static ColumnPtr getColumn(const DictionaryAttribute & dictionary_attribute, size_t size) static ColumnPtr getColumn(const DictionaryAttribute & dictionary_attribute, size_t size)
{ {
if constexpr (std::is_same_v<DictionaryAttributeType, Array>)
{
if (const auto * array_type = typeid_cast<const DataTypeArray *>(dictionary_attribute.type.get()))
{
auto nested_column = array_type->getNestedType()->createColumn();
return ColumnArray::create(std::move(nested_column));
}
else
{
throw Exception(ErrorCodes::TYPE_MISMATCH, "Unsupported attribute type.");
}
}
if constexpr (std::is_same_v<DictionaryAttributeType, String>) if constexpr (std::is_same_v<DictionaryAttributeType, String>)
{ {
return ColumnType::create(); return ColumnType::create();
@ -249,7 +265,8 @@ public:
} }
else if constexpr (IsDecimalNumber<DictionaryAttributeType>) else if constexpr (IsDecimalNumber<DictionaryAttributeType>)
{ {
auto scale = getDecimalScale(*dictionary_attribute.nested_type); auto nested_type = removeNullable(dictionary_attribute.type);
auto scale = getDecimalScale(*nested_type);
return ColumnType::create(size, scale); return ColumnType::create(size, scale);
} }
else if constexpr (is_arithmetic_v<DictionaryAttributeType>) else if constexpr (is_arithmetic_v<DictionaryAttributeType>)
@ -280,18 +297,18 @@ public:
: default_value(std::move(attribute_default_value)) : default_value(std::move(attribute_default_value))
{ {
if (default_values_column_ == nullptr) if (default_values_column_ == nullptr)
use_default_value_from_column = false; use_attribute_default_value = true;
else else
{ {
if (const auto * const default_col = checkAndGetColumn<DefaultColumnType>(*default_values_column_)) if (const auto * const default_col = checkAndGetColumn<DefaultColumnType>(*default_values_column_))
{ {
default_values_column = default_col; default_values_column = default_col;
use_default_value_from_column = true; use_attribute_default_value = false;
} }
else if (const auto * const default_col_const = checkAndGetColumnConst<DefaultColumnType>(default_values_column_.get())) else if (const auto * const default_col_const = checkAndGetColumnConst<DefaultColumnType>(default_values_column_.get()))
{ {
default_value = default_col_const->template getValue<DictionaryAttributeType>(); default_value = default_col_const->template getValue<DictionaryAttributeType>();
use_default_value_from_column = false; use_attribute_default_value = true;
} }
else else
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type of default column is not the same as dictionary attribute type."); throw Exception(ErrorCodes::TYPE_MISMATCH, "Type of default column is not the same as dictionary attribute type.");
@ -300,12 +317,17 @@ public:
DefaultValueType operator[](size_t row) DefaultValueType operator[](size_t row)
{ {
if (!use_default_value_from_column) if (use_attribute_default_value)
return static_cast<DefaultValueType>(default_value); return static_cast<DefaultValueType>(default_value);
assert(default_values_column != nullptr); assert(default_values_column != nullptr);
if constexpr (std::is_same_v<DefaultColumnType, ColumnString>) if constexpr (std::is_same_v<DefaultColumnType, ColumnArray>)
{
Field field = (*default_values_column)[row];
return field.get<Array>();
}
else if constexpr (std::is_same_v<DefaultColumnType, ColumnString>)
return default_values_column->getDataAt(row); return default_values_column->getDataAt(row);
else else
return default_values_column->getData()[row]; return default_values_column->getData()[row];
@ -313,7 +335,7 @@ public:
private: private:
DictionaryAttributeType default_value; DictionaryAttributeType default_value;
const DefaultColumnType * default_values_column = nullptr; const DefaultColumnType * default_values_column = nullptr;
bool use_default_value_from_column = false; bool use_attribute_default_value = false;
}; };
template <DictionaryKeyType key_type> template <DictionaryKeyType key_type>

5
src/Dictionaries/DictionarySourceFactory.cpp
View File

@ -25,9 +25,10 @@ namespace
Block block; Block block;
if (dict_struct.id) if (dict_struct.id)
{
block.insert(ColumnWithTypeAndName{ColumnUInt64::create(1, 0), std::make_shared<DataTypeUInt64>(), dict_struct.id->name}); block.insert(ColumnWithTypeAndName{ColumnUInt64::create(1, 0), std::make_shared<DataTypeUInt64>(), dict_struct.id->name});
}
if (dict_struct.key) else if (dict_struct.key)
{ {
for (const auto & attribute : *dict_struct.key) for (const auto & attribute : *dict_struct.key)
{ {

37
src/Dictionaries/DictionaryStructure.cpp
View File

@ -79,9 +79,7 @@ AttributeUnderlyingType getAttributeUnderlyingType(const DataTypePtr & type)
case TypeIndex::String: return AttributeUnderlyingType::String; case TypeIndex::String: return AttributeUnderlyingType::String;
// Temporary hack to allow arrays in keys, since they are never retrieved for polygon dictionaries. case TypeIndex::Array: return AttributeUnderlyingType::Array;
// TODO: This should be fixed by fully supporting arrays in dictionaries.
case TypeIndex::Array: return AttributeUnderlyingType::String;
default: break; default: break;
} }
@ -125,7 +123,7 @@ DictionaryStructure::DictionaryStructure(const Poco::Util::AbstractConfiguration
id.emplace(config, structure_prefix + ".id"); id.emplace(config, structure_prefix + ".id");
else if (has_key) else if (has_key)
{ {
key.emplace(getAttributes(config, structure_prefix + ".key", true)); key.emplace(getAttributes(config, structure_prefix + ".key", /*complex_key_attributes =*/ true));
if (key->empty()) if (key->empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Empty 'key' supplied"); throw Exception(ErrorCodes::BAD_ARGUMENTS, "Empty 'key' supplied");
} }
@ -173,7 +171,7 @@ DictionaryStructure::DictionaryStructure(const Poco::Util::AbstractConfiguration
has_expressions = true; has_expressions = true;
} }
attributes = getAttributes(config, structure_prefix, false); attributes = getAttributes(config, structure_prefix, /*complex_key_attributes =*/ false);
for (size_t i = 0; i < attributes.size(); ++i) for (size_t i = 0; i < attributes.size(); ++i)
{ {
@ -375,17 +373,10 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
const auto type_string = config.getString(prefix + "type"); const auto type_string = config.getString(prefix + "type");
const auto initial_type = DataTypeFactory::instance().get(type_string); const auto initial_type = DataTypeFactory::instance().get(type_string);
auto type = initial_type; bool is_nullable = initial_type->isNullable();
bool is_array = false;
bool is_nullable = false;
if (type->isNullable()) auto non_nullable_type = removeNullable(initial_type);
{ const auto underlying_type = getAttributeUnderlyingType(non_nullable_type);
is_nullable = true;
type = removeNullable(type);
}
const auto underlying_type = getAttributeUnderlyingType(type);
const auto expression = config.getString(prefix + "expression", ""); const auto expression = config.getString(prefix + "expression", "");
if (!expression.empty()) if (!expression.empty())
@ -394,26 +385,27 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
Field null_value; Field null_value;
if (allow_null_values) if (allow_null_values)
{ {
/// TODO: Fix serialization for nullable type.
const auto null_value_string = config.getString(prefix + "null_value"); const auto null_value_string = config.getString(prefix + "null_value");
try try
{ {
if (null_value_string.empty()) if (null_value_string.empty())
{ {
null_value = type->getDefault(); null_value = non_nullable_type->getDefault();
} }
else else
{ {
ReadBufferFromString null_value_buffer{null_value_string}; ReadBufferFromString null_value_buffer{null_value_string};
auto column_with_null_value = type->createColumn(); auto column_with_null_value = non_nullable_type->createColumn();
type->getDefaultSerialization()->deserializeTextEscaped(*column_with_null_value, null_value_buffer, format_settings); non_nullable_type->getDefaultSerialization()->deserializeTextEscaped(*column_with_null_value, null_value_buffer, format_settings);
null_value = (*column_with_null_value)[0]; null_value = (*column_with_null_value)[0];
} }
} }
catch (Exception & e) catch (Exception & e)
{ {
String dictionary_name = config.getString(".dictionary.name", ""); String dictionary_name = config.getString(".dictionary.name", "");
e.addMessage("While parsing null_value for attribute with name " + name e.addMessage(fmt::format("While parsing null_value for attribute with name {} in dictionary {}", name, dictionary_name));
+ " in dictionary " + dictionary_name);
throw; throw;
} }
} }
@ -436,15 +428,12 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
name, name,
underlying_type, underlying_type,
initial_type, initial_type,
initial_type->getDefaultSerialization(),
type,
expression, expression,
null_value, null_value,
hierarchical, hierarchical,
injective, injective,
is_object_id, is_object_id,
is_nullable, is_nullable});
is_array});
} }
return res_attributes; return res_attributes;

6
src/Dictionaries/DictionaryStructure.h
View File

@ -39,6 +39,7 @@
M(Decimal256) \ M(Decimal256) \
M(UUID) \ M(UUID) \
M(String) \ M(String) \
M(Array) \
namespace DB namespace DB
@ -74,15 +75,12 @@ struct DictionaryAttribute final
const std::string name; const std::string name;
const AttributeUnderlyingType underlying_type; const AttributeUnderlyingType underlying_type;
const DataTypePtr type; const DataTypePtr type;
const SerializationPtr serialization;
const DataTypePtr nested_type;
const std::string expression; const std::string expression;
const Field null_value; const Field null_value;
const bool hierarchical; const bool hierarchical;
const bool injective; const bool injective;
const bool is_object_id; const bool is_object_id;
const bool is_nullable; const bool is_nullable;
const bool is_array;
}; };
template <typename Type> template <typename Type>
@ -92,7 +90,7 @@ struct DictionaryAttributeType
}; };
template <typename F> template <typename F>
void callOnDictionaryAttributeType(AttributeUnderlyingType type, F&& func) void callOnDictionaryAttributeType(AttributeUnderlyingType type, F && func)
{ {
switch (type) switch (type)
{ {

6
src/Dictionaries/ExternalQueryBuilder.cpp
View File

@ -358,7 +358,8 @@ void ExternalQueryBuilder::composeKeyCondition(const Columns & key_columns, cons
/// key_i=value_i /// key_i=value_i
writeQuoted(key_description.name, out); writeQuoted(key_description.name, out);
writeString("=", out); writeString("=", out);
key_description.serialization->serializeTextQuoted(*key_columns[i], row, out, format_settings); auto serialization = key_description.type->getDefaultSerialization();
serialization->serializeTextQuoted(*key_columns[i], row, out, format_settings);
} }
} }
@ -415,7 +416,8 @@ void ExternalQueryBuilder::composeKeyTuple(const Columns & key_columns, const si
writeString(", ", out); writeString(", ", out);
first = false; first = false;
(*dict_struct.key)[i].serialization->serializeTextQuoted(*key_columns[i], row, out, format_settings); auto serialization = (*dict_struct.key)[i].type->getDefaultSerialization();
serialization->serializeTextQuoted(*key_columns[i], row, out, format_settings);
} }
writeString(")", out); writeString(")", out);

48
src/Dictionaries/FlatDictionary.cpp
View File

@ -70,17 +70,27 @@ ColumnPtr FlatDictionary::getColumn(
using ValueType = DictionaryValueType<AttributeType>; using ValueType = DictionaryValueType<AttributeType>;
using ColumnProvider = DictionaryAttributeColumnProvider<AttributeType>; using ColumnProvider = DictionaryAttributeColumnProvider<AttributeType>;
const auto attribute_null_value = std::get<ValueType>(attribute.null_values); const auto & attribute_null_value = std::get<ValueType>(attribute.null_values);
AttributeType null_value = static_cast<AttributeType>(attribute_null_value); AttributeType null_value = static_cast<AttributeType>(attribute_null_value);
DictionaryDefaultValueExtractor<AttributeType> default_value_extractor(std::move(null_value), default_values_column); DictionaryDefaultValueExtractor<AttributeType> default_value_extractor(std::move(null_value), default_values_column);
auto column = ColumnProvider::getColumn(dictionary_attribute, size); auto column = ColumnProvider::getColumn(dictionary_attribute, size);
if constexpr (std::is_same_v<ValueType, StringRef>) if constexpr (std::is_same_v<ValueType, Array>)
{ {
auto * out = column.get(); auto * out = column.get();
getItemsImpl<ValueType, ValueType>( getItemsImpl<ValueType>(
attribute,
ids,
[&](const size_t, const Array & value) { out->insert(value); },
default_value_extractor);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
{
auto * out = column.get();
getItemsImpl<ValueType>(
attribute, attribute,
ids, ids,
[&](const size_t, const StringRef value) { out->insertData(value.data, value.size); }, [&](const size_t, const StringRef value) { out->insertData(value.data, value.size); },
@ -90,7 +100,7 @@ ColumnPtr FlatDictionary::getColumn(
{ {
auto & out = column->getData(); auto & out = column->getData();
getItemsImpl<ValueType, ValueType>( getItemsImpl<ValueType>(
attribute, attribute,
ids, ids,
[&](const size_t row, const auto value) { out[row] = value; }, [&](const size_t row, const auto value) { out[row] = value; },
@ -275,6 +285,7 @@ void FlatDictionary::blockToAttributes(const Block & block)
if (already_processed_keys.find(key) != nullptr) if (already_processed_keys.find(key) != nullptr)
continue; continue;
already_processed_keys.insert(key); already_processed_keys.insert(key);
setAttributeValue(attribute, key, attribute_column[i]); setAttributeValue(attribute, key, attribute_column[i]);
@ -352,7 +363,18 @@ void FlatDictionary::calculateBytesAllocated()
using ValueType = DictionaryValueType<AttributeType>; using ValueType = DictionaryValueType<AttributeType>;
const auto & container = std::get<ContainerType<ValueType>>(attribute.container); const auto & container = std::get<ContainerType<ValueType>>(attribute.container);
bytes_allocated += sizeof(PaddedPODArray<ValueType>) + container.allocated_bytes(); bytes_allocated += sizeof(ContainerType<ValueType>);
if constexpr (std::is_same_v<ValueType, Array>)
{
/// It is not accurate calculations
bytes_allocated += sizeof(Array) * container.size();
}
else
{
bytes_allocated += container.allocated_bytes();
}
bucket_count = container.capacity(); bucket_count = container.capacity();
if constexpr (std::is_same_v<ValueType, StringRef>) if constexpr (std::is_same_v<ValueType, StringRef>)
@ -396,7 +418,7 @@ FlatDictionary::Attribute FlatDictionary::createAttribute(const DictionaryAttrib
return attribute; return attribute;
} }
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueSetter, typename DefaultValueExtractor>
void FlatDictionary::getItemsImpl( void FlatDictionary::getItemsImpl(
const Attribute & attribute, const Attribute & attribute,
const PaddedPODArray<UInt64> & keys, const PaddedPODArray<UInt64> & keys,
@ -414,7 +436,7 @@ void FlatDictionary::getItemsImpl(
if (key < loaded_keys.size() && loaded_keys[key]) if (key < loaded_keys.size() && loaded_keys[key])
{ {
set_value(row, static_cast<OutputType>(container[key])); set_value(row, container[key]);
++keys_found; ++keys_found;
} }
else else
@ -440,7 +462,11 @@ void FlatDictionary::resize(Attribute & attribute, UInt64 key)
{ {
const size_t elements_count = key + 1; //id=0 -> elements_count=1 const size_t elements_count = key + 1; //id=0 -> elements_count=1
loaded_keys.resize(elements_count, false); loaded_keys.resize(elements_count, false);
container.resize_fill(elements_count, std::get<T>(attribute.null_values));
if constexpr (std::is_same_v<T, Array>)
container.resize(elements_count, std::get<T>(attribute.null_values));
else
container.resize_fill(elements_count, std::get<T>(attribute.null_values));
} }
} }
@ -461,13 +487,13 @@ void FlatDictionary::setAttributeValueImpl<String>(Attribute & attribute, UInt64
void FlatDictionary::setAttributeValue(Attribute & attribute, const UInt64 key, const Field & value) void FlatDictionary::setAttributeValue(Attribute & attribute, const UInt64 key, const Field & value)
{ {
auto type_call = [&](const auto &dictionary_attribute_type) auto type_call = [&](const auto & dictionary_attribute_type)
{ {
using Type = std::decay_t<decltype(dictionary_attribute_type)>; using Type = std::decay_t<decltype(dictionary_attribute_type)>;
using AttributeType = typename Type::AttributeType; using AttributeType = typename Type::AttributeType;
using ResizeType = std::conditional_t<std::is_same_v<AttributeType, String>, StringRef, AttributeType>; using ValueType = DictionaryValueType<AttributeType>;
resize<ResizeType>(attribute, key); resize<ValueType>(attribute, key);
if (attribute.nullable_set) if (attribute.nullable_set)
{ {

11
src/Dictionaries/FlatDictionary.h
View File

@ -106,7 +106,7 @@ public:
private: private:
template <typename Value> template <typename Value>
using ContainerType = PaddedPODArray<Value>; using ContainerType = std::conditional_t<std::is_same_v<Value, Array>, std::vector<Value>, PaddedPODArray<Value>>;
using NullableSet = HashSet<UInt64, DefaultHash<UInt64>>; using NullableSet = HashSet<UInt64, DefaultHash<UInt64>>;
@ -135,8 +135,10 @@ private:
Float32, Float32,
Float64, Float64,
UUID, UUID,
StringRef> StringRef,
Array>
null_values; null_values;
std::variant< std::variant<
ContainerType<UInt8>, ContainerType<UInt8>,
ContainerType<UInt16>, ContainerType<UInt16>,
@ -157,7 +159,8 @@ private:
ContainerType<Float32>, ContainerType<Float32>,
ContainerType<Float64>, ContainerType<Float64>,
ContainerType<UUID>, ContainerType<UUID>,
ContainerType<StringRef>> ContainerType<StringRef>,
ContainerType<Array>>
container; container;
std::unique_ptr<Arena> string_arena; std::unique_ptr<Arena> string_arena;
@ -172,7 +175,7 @@ private:
Attribute createAttribute(const DictionaryAttribute& attribute, const Field & null_value); Attribute createAttribute(const DictionaryAttribute& attribute, const Field & null_value);
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueSetter, typename DefaultValueExtractor>
void getItemsImpl( void getItemsImpl(
const Attribute & attribute, const Attribute & attribute,
const PaddedPODArray<UInt64> & keys, const PaddedPODArray<UInt64> & keys,

17
src/Dictionaries/HashedDictionary.cpp
View File

@ -90,13 +90,26 @@ ColumnPtr HashedDictionary<dictionary_key_type, sparse>::getColumn(
using ValueType = DictionaryValueType<AttributeType>; using ValueType = DictionaryValueType<AttributeType>;
using ColumnProvider = DictionaryAttributeColumnProvider<AttributeType>; using ColumnProvider = DictionaryAttributeColumnProvider<AttributeType>;
const auto attribute_null_value = std::get<ValueType>(attribute.null_values); const auto & attribute_null_value = std::get<ValueType>(attribute.null_values);
AttributeType null_value = static_cast<AttributeType>(attribute_null_value); AttributeType null_value = static_cast<AttributeType>(attribute_null_value);
DictionaryDefaultValueExtractor<AttributeType> default_value_extractor(std::move(null_value), default_values_column); DictionaryDefaultValueExtractor<AttributeType> default_value_extractor(std::move(null_value), default_values_column);
auto column = ColumnProvider::getColumn(dictionary_attribute, size); auto column = ColumnProvider::getColumn(dictionary_attribute, size);
if constexpr (std::is_same_v<ValueType, StringRef>) if constexpr (std::is_same_v<ValueType, Array>)
{
auto * out = column.get();
getItemsImpl<ValueType>(
attribute,
extractor,
[&](const size_t, const Array & value) { out->insert(value); },
[&](const size_t)
{
},
default_value_extractor);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
{ {
auto * out = column.get(); auto * out = column.get();

6
src/Dictionaries/HashedDictionary.h
View File

@ -173,7 +173,8 @@ private:
Float32, Float32,
Float64, Float64,
UUID, UUID,
StringRef> StringRef,
Array>
null_values; null_values;
std::variant< std::variant<
@ -196,7 +197,8 @@ private:
CollectionType<Float32>, CollectionType<Float32>,
CollectionType<Float64>, CollectionType<Float64>,
CollectionType<UUID>, CollectionType<UUID>,
CollectionType<StringRef>> CollectionType<StringRef>,
CollectionType<Array>>
container; container;
std::unique_ptr<Arena> string_arena; std::unique_ptr<Arena> string_arena;

38
src/Dictionaries/IPAddressDictionary.cpp
View File

@ -27,6 +27,7 @@ namespace ErrorCodes
extern const int DICTIONARY_IS_EMPTY; extern const int DICTIONARY_IS_EMPTY;
extern const int LOGICAL_ERROR; extern const int LOGICAL_ERROR;
extern const int TYPE_MISMATCH; extern const int TYPE_MISMATCH;
extern const int UNSUPPORTED_METHOD;
} }
namespace namespace
@ -237,11 +238,21 @@ ColumnPtr IPAddressDictionary::getColumn(
auto column = ColumnProvider::getColumn(dictionary_attribute, size); auto column = ColumnProvider::getColumn(dictionary_attribute, size);
if constexpr (std::is_same_v<AttributeType, String>) if constexpr (std::is_same_v<ValueType, Array>)
{ {
auto * out = column.get(); auto * out = column.get();
getItemsImpl<ValueType, ValueType>( getItemsImpl<ValueType>(
attribute,
key_columns,
[&](const size_t, const Array & value) { out->insert(value); },
default_value_extractor);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
{
auto * out = column.get();
getItemsImpl<ValueType>(
attribute, attribute,
key_columns, key_columns,
[&](const size_t, const StringRef value) { out->insertData(value.data, value.size); }, [&](const size_t, const StringRef value) { out->insertData(value.data, value.size); },
@ -251,7 +262,7 @@ ColumnPtr IPAddressDictionary::getColumn(
{ {
auto & out = column->getData(); auto & out = column->getData();
getItemsImpl<ValueType, ValueType>( getItemsImpl<ValueType>(
attribute, attribute,
key_columns, key_columns,
[&](const size_t row, const auto value) { return out[row] = value; }, [&](const size_t row, const auto value) { return out[row] = value; },
@ -315,8 +326,15 @@ void IPAddressDictionary::createAttributes()
auto create_attributes_from_dictionary_attributes = [this](const std::vector<DictionaryAttribute> & dict_attrs) auto create_attributes_from_dictionary_attributes = [this](const std::vector<DictionaryAttribute> & dict_attrs)
{ {
attributes.reserve(attributes.size() + dict_attrs.size()); attributes.reserve(attributes.size() + dict_attrs.size());
for (const auto & attribute : dict_attrs) for (const auto & attribute : dict_attrs)
{ {
if (attribute.is_nullable)
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
"{}: array or nullable attributes not supported for dictionary of type {}",
full_name,
getTypeName());
attribute_index_by_name.emplace(attribute.name, attributes.size()); attribute_index_by_name.emplace(attribute.name, attributes.size());
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value)); attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
@ -583,7 +601,7 @@ const uint8_t * IPAddressDictionary::getIPv6FromOffset(const IPAddressDictionary
return reinterpret_cast<const uint8_t *>(&ipv6_col[i * IPV6_BINARY_LENGTH]); return reinterpret_cast<const uint8_t *>(&ipv6_col[i * IPV6_BINARY_LENGTH]);
} }
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueSetter, typename DefaultValueExtractor>
void IPAddressDictionary::getItemsByTwoKeyColumnsImpl( void IPAddressDictionary::getItemsByTwoKeyColumnsImpl(
const Attribute & attribute, const Attribute & attribute,
const Columns & key_columns, const Columns & key_columns,
@ -622,7 +640,7 @@ void IPAddressDictionary::getItemsByTwoKeyColumnsImpl(
(*ipv4_col)[*found_it] == addr && (*ipv4_col)[*found_it] == addr &&
mask_column[*found_it] == mask)) mask_column[*found_it] == mask))
{ {
set_value(i, static_cast<OutputType>(vec[row_idx[*found_it]])); set_value(i, vec[row_idx[*found_it]]);
} }
else else
set_value(i, default_value_extractor[i]); set_value(i, default_value_extractor[i]);
@ -658,13 +676,13 @@ void IPAddressDictionary::getItemsByTwoKeyColumnsImpl(
if (likely(found_it != range.end() && if (likely(found_it != range.end() &&
memequal16(getIPv6FromOffset(*ipv6_col, *found_it), target.addr) && memequal16(getIPv6FromOffset(*ipv6_col, *found_it), target.addr) &&
mask_column[*found_it] == mask)) mask_column[*found_it] == mask))
set_value(i, static_cast<OutputType>(vec[row_idx[*found_it]])); set_value(i, vec[row_idx[*found_it]]);
else else
set_value(i, default_value_extractor[i]); set_value(i, default_value_extractor[i]);
} }
} }
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueSetter, typename DefaultValueExtractor>
void IPAddressDictionary::getItemsImpl( void IPAddressDictionary::getItemsImpl(
const Attribute & attribute, const Attribute & attribute,
const Columns & key_columns, const Columns & key_columns,
@ -677,7 +695,7 @@ void IPAddressDictionary::getItemsImpl(
// special case for getBlockInputStream // special case for getBlockInputStream
if (unlikely(key_columns.size() == 2)) if (unlikely(key_columns.size() == 2))
{ {
getItemsByTwoKeyColumnsImpl<AttributeType, OutputType>( getItemsByTwoKeyColumnsImpl<AttributeType>(
attribute, key_columns, std::forward<ValueSetter>(set_value), default_value_extractor); attribute, key_columns, std::forward<ValueSetter>(set_value), default_value_extractor);
query_count.fetch_add(rows, std::memory_order_relaxed); query_count.fetch_add(rows, std::memory_order_relaxed);
return; return;
@ -697,7 +715,7 @@ void IPAddressDictionary::getItemsImpl(
auto found = tryLookupIPv4(addrv4, addrv6_buf); auto found = tryLookupIPv4(addrv4, addrv6_buf);
if (found != ipNotFound()) if (found != ipNotFound())
{ {
set_value(i, static_cast<OutputType>(vec[*found])); set_value(i, vec[*found]);
++keys_found; ++keys_found;
} }
else else
@ -715,7 +733,7 @@ void IPAddressDictionary::getItemsImpl(
auto found = tryLookupIPv6(reinterpret_cast<const uint8_t *>(addr.data)); auto found = tryLookupIPv6(reinterpret_cast<const uint8_t *>(addr.data));
if (found != ipNotFound()) if (found != ipNotFound())
{ {
set_value(i, static_cast<OutputType>(vec[*found])); set_value(i, vec[*found]);
++keys_found; ++keys_found;
} }
else else

10
src/Dictionaries/IPAddressDictionary.h
View File

@ -116,7 +116,8 @@ private:
Float32, Float32,
Float64, Float64,
UUID, UUID,
String> String,
Array>
null_values; null_values;
std::variant< std::variant<
ContainerType<UInt8>, ContainerType<UInt8>,
@ -138,7 +139,8 @@ private:
ContainerType<Float32>, ContainerType<Float32>,
ContainerType<Float64>, ContainerType<Float64>,
ContainerType<UUID>, ContainerType<UUID>,
ContainerType<StringRef>> ContainerType<StringRef>,
ContainerType<Array>>
maps; maps;
std::unique_ptr<Arena> string_arena; std::unique_ptr<Arena> string_arena;
}; };
@ -157,14 +159,14 @@ private:
static Attribute createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value); static Attribute createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value);
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueSetter, typename DefaultValueExtractor>
void getItemsByTwoKeyColumnsImpl( void getItemsByTwoKeyColumnsImpl(
const Attribute & attribute, const Attribute & attribute,
const Columns & key_columns, const Columns & key_columns,
ValueSetter && set_value, ValueSetter && set_value,
DefaultValueExtractor & default_value_extractor) const; DefaultValueExtractor & default_value_extractor) const;
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType,typename ValueSetter, typename DefaultValueExtractor>
void getItemsImpl( void getItemsImpl(
const Attribute & attribute, const Attribute & attribute,
const Columns & key_columns, const Columns & key_columns,

137
src/Dictionaries/PolygonDictionary.cpp
View File

@ -52,7 +52,6 @@ ColumnPtr IPolygonDictionary::getColumn(
const auto requested_key_points = extractPoints(key_columns); const auto requested_key_points = extractPoints(key_columns);
const auto & attribute = dict_struct.getAttribute(attribute_name, result_type); const auto & attribute = dict_struct.getAttribute(attribute_name, result_type);
bool complex_attribute = attribute.is_nullable || attribute.is_array;
DefaultValueProvider default_value_provider(attribute.null_value, default_values_column); DefaultValueProvider default_value_provider(attribute.null_value, default_values_column);
size_t attribute_index = dict_struct.attribute_name_to_index.find(attribute_name)->second; size_t attribute_index = dict_struct.attribute_name_to_index.find(attribute_name)->second;
@ -61,28 +60,13 @@ ColumnPtr IPolygonDictionary::getColumn(
auto result = attribute_values_column->cloneEmpty(); auto result = attribute_values_column->cloneEmpty();
result->reserve(requested_key_points.size()); result->reserve(requested_key_points.size());
Field row_value_to_insert; if (unlikely(attribute.is_nullable))
size_t polygon_index = 0;
size_t keys_found = 0;
if (unlikely(complex_attribute))
{ {
for (size_t requested_key_index = 0; requested_key_index < requested_key_points.size(); ++requested_key_index) getItemsImpl<Field>(
{ requested_key_points,
const auto found = find(requested_key_points[requested_key_index], polygon_index); [&](size_t row) { return (*attribute_values_column)[row]; },
[&](Field & value) { result->insert(value); },
if (found) default_value_provider);
{
size_t attribute_values_index = polygon_index_to_attribute_value_index[polygon_index];
attribute_values_column->get(attribute_values_index, row_value_to_insert);
++keys_found;
}
else
row_value_to_insert = default_value_provider.getDefaultValue(requested_key_index);
result->insert(row_value_to_insert);
}
} }
else else
{ {
@ -91,10 +75,8 @@ ColumnPtr IPolygonDictionary::getColumn(
using Type = std::decay_t<decltype(dictionary_attribute_type)>; using Type = std::decay_t<decltype(dictionary_attribute_type)>;
using AttributeType = typename Type::AttributeType; using AttributeType = typename Type::AttributeType;
using ValueType = DictionaryValueType<AttributeType>; using ValueType = DictionaryValueType<AttributeType>;
using ColumnType = std::conditional_t< using ColumnProvider = DictionaryAttributeColumnProvider<AttributeType>;
std::is_same_v<AttributeType, String>, using ColumnType = typename ColumnProvider::ColumnType;
ColumnString,
std::conditional_t<IsDecimalNumber<AttributeType>, ColumnDecimal<ValueType>, ColumnVector<AttributeType>>>;
const auto attribute_values_column_typed = typeid_cast<const ColumnType *>(attribute_values_column.get()); const auto attribute_values_column_typed = typeid_cast<const ColumnType *>(attribute_values_column.get());
if (!attribute_values_column_typed) if (!attribute_values_column_typed)
@ -102,54 +84,38 @@ ColumnPtr IPolygonDictionary::getColumn(
ColumnType & result_column_typed = static_cast<ColumnType &>(*result); ColumnType & result_column_typed = static_cast<ColumnType &>(*result);
if constexpr (std::is_same_v<ColumnType, ColumnString>) if constexpr (std::is_same_v<ValueType, Array>)
{ {
for (size_t requested_key_index = 0; requested_key_index < requested_key_points.size(); ++requested_key_index) getItemsImpl<ValueType>(
{ requested_key_points,
const auto found = find(requested_key_points[requested_key_index], polygon_index); [&](size_t row) { return (*attribute_values_column)[row].get<Array>(); },
[&](Array & value) { result_column_typed.insert(value); },
if (found) default_value_provider);
{ }
size_t attribute_values_index = polygon_index_to_attribute_value_index[polygon_index]; else if constexpr (std::is_same_v<ValueType, StringRef>)
auto data_to_insert = attribute_values_column->getDataAt(attribute_values_index); {
result_column_typed.insertData(data_to_insert.data, data_to_insert.size); getItemsImpl<ValueType>(
++keys_found; requested_key_points,
} [&](size_t row) { return attribute_values_column->getDataAt(row); },
else [&](StringRef value) { result_column_typed.insertData(value.data, value.size); },
result_column_typed.insert(default_value_provider.getDefaultValue(requested_key_index)); default_value_provider);
}
} }
else else
{ {
auto & attribute_data = attribute_values_column_typed->getData(); auto & attribute_data = attribute_values_column_typed->getData();
auto & result_data = result_column_typed.getData(); auto & result_data = result_column_typed.getData();
for (size_t requested_key_index = 0; requested_key_index < requested_key_points.size(); ++requested_key_index) getItemsImpl<ValueType>(
{ requested_key_points,
const auto found = find(requested_key_points[requested_key_index], polygon_index); [&](size_t row) { return attribute_data[row]; },
[&](auto value) { result_data.emplace_back(value); },
if (found) default_value_provider);
{
size_t attribute_values_index = polygon_index_to_attribute_value_index[polygon_index];
auto & item = attribute_data[attribute_values_index];
result_data.emplace_back(item);
++keys_found;
}
else
{
row_value_to_insert = default_value_provider.getDefaultValue(requested_key_index);
result_data.emplace_back(row_value_to_insert.template get<NearestFieldType<ValueType>>());
}
}
} }
}; };
callOnDictionaryAttributeType(attribute.underlying_type, type_call); callOnDictionaryAttributeType(attribute.underlying_type, type_call);
} }
query_count.fetch_add(requested_key_points.size(), std::memory_order_relaxed);
found_count.fetch_add(keys_found, std::memory_order_relaxed);
return result; return result;
} }
@ -309,6 +275,55 @@ ColumnUInt8::Ptr IPolygonDictionary::hasKeys(const Columns & key_columns, const
return result; return result;
} }
template <typename AttributeType, typename ValueGetter, typename ValueSetter, typename DefaultValueExtractor>
void IPolygonDictionary::getItemsImpl(
const std::vector<IPolygonDictionary::Point> & requested_key_points,
ValueGetter && get_value,
ValueSetter && set_value,
DefaultValueExtractor & default_value_extractor) const
{
size_t polygon_index = 0;
size_t keys_found = 0;
for (size_t requested_key_index = 0; requested_key_index < requested_key_points.size(); ++requested_key_index)
{
const auto found = find(requested_key_points[requested_key_index], polygon_index);
if (found)
{
size_t attribute_values_index = polygon_index_to_attribute_value_index[polygon_index];
auto value = get_value(attribute_values_index);
set_value(value);
++keys_found;
}
else
{
Field default_value = default_value_extractor.getDefaultValue(requested_key_index);
if constexpr (std::is_same_v<AttributeType, Field>)
{
set_value(default_value);
}
else if constexpr (std::is_same_v<AttributeType, Array>)
{
set_value(default_value.get<Array>());
}
else if constexpr (std::is_same_v<AttributeType, StringRef>)
{
auto default_value_string = default_value.get<String>();
set_value(default_value_string);
}
else
{
set_value(default_value.get<NearestFieldType<AttributeType>>());
}
}
}
query_count.fetch_add(requested_key_points.size(), std::memory_order_relaxed);
found_count.fetch_add(keys_found, std::memory_order_relaxed);
}
namespace namespace
{ {

7
src/Dictionaries/PolygonDictionary.h
View File

@ -49,6 +49,7 @@ public:
Array, Array,
Tuple, Tuple,
}; };
IPolygonDictionary( IPolygonDictionary(
const StorageID & dict_id_, const StorageID & dict_id_,
const DictionaryStructure & dict_struct_, const DictionaryStructure & dict_struct_,
@ -138,10 +139,10 @@ private:
size_t getAttributeIndex(const std::string & attribute_name) const; size_t getAttributeIndex(const std::string & attribute_name) const;
/** Helper function for retrieving the value of an attribute by key. */ /** Helper function for retrieving the value of an attribute by key. */
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueGetter, typename ValueSetter, typename DefaultValueExtractor>
void getItemsImpl( void getItemsImpl(
size_t attribute_ind, const std::vector<IPolygonDictionary::Point> & requested_key_points,
const Columns & key_columns, ValueGetter && get_value,
ValueSetter && set_value, ValueSetter && set_value,
DefaultValueExtractor & default_value_extractor) const; DefaultValueExtractor & default_value_extractor) const;

29
src/Dictionaries/RangeHashedDictionary.cpp
View File

@ -125,17 +125,30 @@ ColumnPtr RangeHashedDictionary::getColumn(
using ValueType = DictionaryValueType<AttributeType>; using ValueType = DictionaryValueType<AttributeType>;
using ColumnProvider = DictionaryAttributeColumnProvider<AttributeType>; using ColumnProvider = DictionaryAttributeColumnProvider<AttributeType>;
const auto attribute_null_value = std::get<ValueType>(attribute.null_values); const auto & attribute_null_value = std::get<ValueType>(attribute.null_values);
AttributeType null_value = static_cast<AttributeType>(attribute_null_value); AttributeType null_value = static_cast<AttributeType>(attribute_null_value);
DictionaryDefaultValueExtractor<AttributeType> default_value_extractor(std::move(null_value), default_values_column); DictionaryDefaultValueExtractor<AttributeType> default_value_extractor(std::move(null_value), default_values_column);
auto column = ColumnProvider::getColumn(dictionary_attribute, keys_size); auto column = ColumnProvider::getColumn(dictionary_attribute, keys_size);
if constexpr (std::is_same_v<AttributeType, String>) if constexpr (std::is_same_v<ValueType, Array>)
{ {
auto * out = column.get(); auto * out = column.get();
getItemsImpl<ValueType, ValueType>( getItemsImpl<ValueType>(
attribute,
modified_key_columns,
[&](const size_t, const Array & value, bool)
{
out->insert(value);
},
default_value_extractor);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
{
auto * out = column.get();
getItemsImpl<ValueType>(
attribute, attribute,
modified_key_columns, modified_key_columns,
[&](const size_t row, const StringRef value, bool is_null) [&](const size_t row, const StringRef value, bool is_null)
@ -151,7 +164,7 @@ ColumnPtr RangeHashedDictionary::getColumn(
{ {
auto & out = column->getData(); auto & out = column->getData();
getItemsImpl<ValueType, ValueType>( getItemsImpl<ValueType>(
attribute, attribute,
modified_key_columns, modified_key_columns,
[&](const size_t row, const auto value, bool is_null) [&](const size_t row, const auto value, bool is_null)
@ -369,7 +382,7 @@ void RangeHashedDictionary::createAttributeImpl<String>(Attribute & attribute, c
} }
RangeHashedDictionary::Attribute RangeHashedDictionary::Attribute
RangeHashedDictionary::createAttribute(const DictionaryAttribute& attribute, const Field & null_value) RangeHashedDictionary::createAttribute(const DictionaryAttribute & attribute, const Field & null_value)
{ {
Attribute attr{attribute.underlying_type, attribute.is_nullable, {}, {}, {}}; Attribute attr{attribute.underlying_type, attribute.is_nullable, {}, {}, {}};
@ -385,7 +398,7 @@ RangeHashedDictionary::createAttribute(const DictionaryAttribute& attribute, con
return attr; return attr;
} }
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueSetter, typename DefaultValueExtractor>
void RangeHashedDictionary::getItemsImpl( void RangeHashedDictionary::getItemsImpl(
const Attribute & attribute, const Attribute & attribute,
const Columns & key_columns, const Columns & key_columns,
@ -422,8 +435,8 @@ void RangeHashedDictionary::getItemsImpl(
++keys_found; ++keys_found;
auto & value = val_it->value; auto & value = val_it->value;
if (value) if (value.has_value())
set_value(row, static_cast<OutputType>(*value), false); // NOLINT set_value(row, *value, false);
else else
set_value(row, default_value_extractor[row], true); set_value(row, default_value_extractor[row], true);
} }

8
src/Dictionaries/RangeHashedDictionary.h
View File

@ -127,7 +127,8 @@ private:
Float32, Float32,
Float64, Float64,
UUID, UUID,
StringRef> StringRef,
Array>
null_values; null_values;
std::variant< std::variant<
Ptr<UInt8>, Ptr<UInt8>,
@ -149,7 +150,8 @@ private:
Ptr<Float32>, Ptr<Float32>,
Ptr<Float64>, Ptr<Float64>,
Ptr<UUID>, Ptr<UUID>,
Ptr<StringRef>> Ptr<StringRef>,
Ptr<Array>>
maps; maps;
std::unique_ptr<Arena> string_arena; std::unique_ptr<Arena> string_arena;
}; };
@ -168,7 +170,7 @@ private:
static Attribute createAttribute(const DictionaryAttribute& attribute, const Field & null_value); static Attribute createAttribute(const DictionaryAttribute& attribute, const Field & null_value);
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueSetter, typename DefaultValueExtractor>
void getItemsImpl( void getItemsImpl(
const Attribute & attribute, const Attribute & attribute,
const Columns & key_columns, const Columns & key_columns,

5
src/Parsers/ParserDictionaryAttributeDeclaration.cpp
View File

@ -17,6 +17,7 @@ bool ParserDictionaryAttributeDeclaration::parseImpl(Pos & pos, ASTPtr & node, E
ParserKeyword s_injective{"INJECTIVE"}; ParserKeyword s_injective{"INJECTIVE"};
ParserKeyword s_is_object_id{"IS_OBJECT_ID"}; ParserKeyword s_is_object_id{"IS_OBJECT_ID"};
ParserLiteral default_parser; ParserLiteral default_parser;
ParserArrayOfLiterals array_literals_parser;
ParserTernaryOperatorExpression expression_parser; ParserTernaryOperatorExpression expression_parser;
/// mandatory attribute name /// mandatory attribute name
@ -40,8 +41,10 @@ bool ParserDictionaryAttributeDeclaration::parseImpl(Pos & pos, ASTPtr & node, E
{ {
if (!default_value && s_default.ignore(pos, expected)) if (!default_value && s_default.ignore(pos, expected))
{ {
if (!default_parser.parse(pos, default_value, expected)) if (!default_parser.parse(pos, default_value, expected) &&
!array_literals_parser.parse(pos, default_value, expected))
return false; return false;
continue; continue;
} }

28
tests/queries/0_stateless/01902_dictionary_array_type.reference Normal file
View File

@ -0,0 +1,28 @@
Flat dictionary
[0,1,2]
[1,2,3]
[2,3,4]
Hashed dictionary
[0,1,2]
[1,2,3]
[2,3,4]
Cache dictionary
[0,1,2]
[1,2,3]
[2,3,4]
Direct dictionary
[0,1,2]
[1,2,3]
[2,3,4]
IPTrie dictionary
[0,1,2]
[1,2,3]
[2,3,4]
Polygon dictionary
[0,1,2]
[1,2,3]
[2,3,4]
Range dictionary
[0,1,2]
[1,2,3]
[2,3,4]

164
tests/queries/0_stateless/01902_dictionary_array_type.sql Normal file
View File

@ -0,0 +1,164 @@
DROP TABLE IF EXISTS dictionary_array_source_table;
CREATE TABLE dictionary_array_source_table
(
id UInt64,
array_value Array(Int64)
) ENGINE=TinyLog;
INSERT INTO dictionary_array_source_table VALUES (0, [0, 1, 2]);
DROP DICTIONARY IF EXISTS flat_dictionary;
CREATE DICTIONARY flat_dictionary
(
id UInt64,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'dictionary_array_source_table'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(FLAT());
SELECT 'Flat dictionary';
SELECT dictGet('flat_dictionary', 'array_value', toUInt64(0));
SELECT dictGet('flat_dictionary', 'array_value', toUInt64(1));
SELECT dictGetOrDefault('flat_dictionary', 'array_value', toUInt64(1), [2,3,4]);
DROP DICTIONARY flat_dictionary;
DROP DICTIONARY IF EXISTS hashed_dictionary;
CREATE DICTIONARY hashed_dictionary
(
id UInt64,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'dictionary_array_source_table'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(HASHED());
SELECT 'Hashed dictionary';
SELECT dictGet('hashed_dictionary', 'array_value', toUInt64(0));
SELECT dictGet('hashed_dictionary', 'array_value', toUInt64(1));
SELECT dictGetOrDefault('hashed_dictionary', 'array_value', toUInt64(1), [2,3,4]);
DROP DICTIONARY hashed_dictionary;
DROP DICTIONARY IF EXISTS cache_dictionary;
CREATE DICTIONARY cache_dictionary
(
id UInt64,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'dictionary_array_source_table'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(CACHE(SIZE_IN_CELLS 10));
SELECT 'Cache dictionary';
SELECT dictGet('cache_dictionary', 'array_value', toUInt64(0));
SELECT dictGet('cache_dictionary', 'array_value', toUInt64(1));
SELECT dictGetOrDefault('cache_dictionary', 'array_value', toUInt64(1), [2,3,4]);
DROP DICTIONARY cache_dictionary;
DROP DICTIONARY IF EXISTS direct_dictionary;
CREATE DICTIONARY direct_dictionary
(
id UInt64,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'dictionary_array_source_table'))
LAYOUT(DIRECT());
SELECT 'Direct dictionary';
SELECT dictGet('direct_dictionary', 'array_value', toUInt64(0));
SELECT dictGet('direct_dictionary', 'array_value', toUInt64(1));
SELECT dictGetOrDefault('direct_dictionary', 'array_value', toUInt64(1), [2,3,4]);
DROP DICTIONARY direct_dictionary;
DROP TABLE IF EXISTS ip_trie_dictionary_array_source_table;
CREATE TABLE ip_trie_dictionary_array_source_table
(
prefix String,
array_value Array(Int64)
) ENGINE = TinyLog;
DROP TABLE dictionary_array_source_table;
DROP DICTIONARY IF EXISTS ip_trie_dictionary;
CREATE DICTIONARY ip_trie_dictionary
(
prefix String,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY prefix
SOURCE(CLICKHOUSE(HOST 'localhost' port tcpPort() TABLE 'ip_trie_dictionary_array_source_table'))
LIFETIME(MIN 10 MAX 1000)
LAYOUT(IP_TRIE());
INSERT INTO ip_trie_dictionary_array_source_table VALUES ('127.0.0.0', [0, 1, 2]);
SELECT 'IPTrie dictionary';
SELECT dictGet('ip_trie_dictionary', 'array_value', tuple(IPv4StringToNum('127.0.0.0')));
SELECT dictGet('ip_trie_dictionary', 'array_value', tuple(IPv4StringToNum('128.0.0.0')));
SELECT dictGetOrDefault('ip_trie_dictionary', 'array_value', tuple(IPv4StringToNum('128.0.0.0')), [2,3,4]);
DROP DICTIONARY ip_trie_dictionary;
DROP TABLE ip_trie_dictionary_array_source_table;
DROP TABLE IF EXISTS polygon_dictionary_array_source_table;
CREATE TABLE polygon_dictionary_array_source_table
(
key Array(Array(Array(Tuple(Float64, Float64)))),
array_value Array(Int64)
) ENGINE = TinyLog;
INSERT INTO polygon_dictionary_array_source_table VALUES ([[[(0, 0), (0, 1), (1, 1), (1, 0)]]], [0, 1, 2]);
DROP DICTIONARY IF EXISTS polygon_dictionary;
CREATE DICTIONARY polygon_dictionary
(
key Array(Array(Array(Tuple(Float64, Float64)))),
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY key
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'polygon_dictionary_array_source_table'))
LIFETIME(MIN 0 MAX 1000)
LAYOUT(POLYGON());
SELECT 'Polygon dictionary';
SELECT dictGet('polygon_dictionary', 'array_value', tuple(0.5, 0.5));
SELECT dictGet('polygon_dictionary', 'array_value', tuple(1.5, 1.5));
SELECT dictGetOrDefault('polygon_dictionary', 'array_value', tuple(1.5, 1.5), [2, 3, 4]);
DROP DICTIONARY polygon_dictionary;
DROP TABLE polygon_dictionary_array_source_table;
CREATE TABLE range_dictionary_array_source_table
(
key UInt64,
start_date Date,
end_date Date,
array_value Array(Int64)
)
ENGINE = TinyLog;
INSERT INTO range_dictionary_array_source_table VALUES(1, toDate('2019-05-05'), toDate('2019-05-20'), [0, 1, 2]);
CREATE DICTIONARY range_dictionary
(
key UInt64,
start_date Date,
end_date Date,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY key
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'range_dictionary_array_source_table'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(RANGE_HASHED())
RANGE(MIN start_date MAX end_date);
SELECT 'Range dictionary';
SELECT dictGet('range_dictionary', 'array_value', toUInt64(1), toDate('2019-05-15'));
SELECT dictGet('range_dictionary', 'array_value', toUInt64(1), toDate('2019-05-21'));
SELECT dictGetOrDefault('range_dictionary', 'array_value', toUInt64(1), toDate('2019-05-21'), [2, 3, 4]);
DROP DICTIONARY range_dictionary;
DROP TABLE range_dictionary_array_source_table;

4
tests/queries/0_stateless/01903_ssd_cache_dictionary_array_type.reference Normal file
View File

@ -0,0 +1,4 @@
SSDCache dictionary
[0,1,2]
[1,2,3]
[2,3,4]

35
tests/queries/0_stateless/01903_ssd_cache_dictionary_array_type.sh Executable file
View File

@ -0,0 +1,35 @@
#!/usr/bin/env bash
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
USER_FILES_PATH=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}')
$CLICKHOUSE_CLIENT -n --query="
DROP TABLE IF EXISTS dictionary_array_source_table;
CREATE TABLE dictionary_array_source_table
(
id UInt64,
array_value Array(Int64)
) ENGINE=TinyLog;
INSERT INTO dictionary_array_source_table VALUES (0, [0, 1, 2]);
DROP DICTIONARY IF EXISTS ssd_cache_dictionary;
CREATE DICTIONARY ssd_cache_dictionary
(
id UInt64,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'dictionary_array_source_table'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 8192 PATH '$USER_FILES_PATH/0d'));
SELECT 'SSDCache dictionary';
SELECT dictGet('ssd_cache_dictionary', 'array_value', toUInt64(0));
SELECT dictGet('ssd_cache_dictionary', 'array_value', toUInt64(1));
SELECT dictGetOrDefault('ssd_cache_dictionary', 'array_value', toUInt64(1), [2,3,4]);
DROP DICTIONARY ssd_cache_dictionary;
DROP TABLE dictionary_array_source_table;"

3
tests/queries/skip_list.json
View File

@ -412,7 +412,8 @@
"01684_ssd_cache_dictionary_simple_key", "01684_ssd_cache_dictionary_simple_key",
"01685_ssd_cache_dictionary_complex_key", "01685_ssd_cache_dictionary_complex_key",
"01889_postgresql_protocol_null_fields", "01889_postgresql_protocol_null_fields",
"01889_check_row_policy_defined_using_user_function" "01889_check_row_policy_defined_using_user_function",
"01903_ssd_cache_dictionary_array_type"
], ],
"parallel": "parallel":
[ [