Added tests

This commit is contained in:
Maksim Kita 2021-06-09 20:21:32 +03:00 committed by Maksim Kita
parent 45b8dc772b
commit eabeb3076f
15 changed files with 478 additions and 217 deletions

View File

@ -226,23 +226,17 @@ private:
auto & attribute = attributes[attribute_index]; auto & attribute = attributes[attribute_index];
const auto & default_value_provider = fetch_request.defaultValueProviderAtIndex(attribute_index); const auto & default_value_provider = fetch_request.defaultValueProviderAtIndex(attribute_index);
size_t fetched_keys_size = fetched_keys.size();
auto & fetched_column = *result.fetched_columns[attribute_index]; auto & fetched_column = *result.fetched_columns[attribute_index];
fetched_column.reserve(fetched_keys_size); fetched_column.reserve(fetched_columns_index);
if (unlikely(attribute.is_complex_type)) if (unlikely(attribute.is_complex_type))
{ {
auto & container = std::get<std::vector<Field>>(attribute.attribute_container); getItemsForFetchedKeys<Field>(
attribute,
for (size_t fetched_key_index = 0; fetched_key_index < fetched_columns_index; ++fetched_key_index) fetched_columns_index,
{ fetched_keys,
auto fetched_key = fetched_keys[fetched_key_index]; [&](Field & value) { fetched_column.insert(value); },
default_value_provider);
if (unlikely(fetched_key.is_default))
fetched_column.insert(default_value_provider.getDefaultValue(fetched_key_index));
else
fetched_column.insert(container[fetched_key.element_index]);
}
} }
else else
{ {
@ -250,46 +244,40 @@ private:
{ {
using Type = std::decay_t<decltype(dictionary_attribute_type)>; using Type = std::decay_t<decltype(dictionary_attribute_type)>;
using AttributeType = typename Type::AttributeType; using AttributeType = typename Type::AttributeType;
using ColumnProvider = DictionaryAttributeColumnProvider<AttributeType>;
using ColumnType = typename ColumnProvider::ColumnType;
using ValueType = DictionaryValueType<AttributeType>; using ValueType = DictionaryValueType<AttributeType>;
using ColumnType =
std::conditional_t<std::is_same_v<AttributeType, String>, ColumnString,
std::conditional_t<IsDecimalNumber<AttributeType>, ColumnDecimal<ValueType>,
ColumnVector<AttributeType>>>;
auto & container = std::get<PaddedPODArray<ValueType>>(attribute.attribute_container);
ColumnType & column_typed = static_cast<ColumnType &>(fetched_column); ColumnType & column_typed = static_cast<ColumnType &>(fetched_column);
if constexpr (std::is_same_v<ColumnType, ColumnString>) if constexpr (std::is_same_v<ValueType, Array>)
{ {
for (size_t fetched_key_index = 0; fetched_key_index < fetched_columns_index; ++fetched_key_index) getItemsForFetchedKeys<ValueType>(
{ attribute,
auto fetched_key = fetched_keys[fetched_key_index]; fetched_columns_index,
fetched_keys,
if (unlikely(fetched_key.is_default)) [&](Array & value) { fetched_column.insert(value); },
column_typed.insert(default_value_provider.getDefaultValue(fetched_key_index)); default_value_provider);
else
{
auto item = container[fetched_key.element_index];
column_typed.insertData(item.data, item.size);
}
} }
else if constexpr (std::is_same_v<ValueType, StringRef>)
{
getItemsForFetchedKeys<ValueType>(
attribute,
fetched_columns_index,
fetched_keys,
[&](StringRef value) { fetched_column.insertData(value.data, value.size); },
default_value_provider);
} }
else else
{ {
auto & data = column_typed.getData(); auto & data = column_typed.getData();
for (size_t fetched_key_index = 0; fetched_key_index < fetched_columns_index; ++fetched_key_index) getItemsForFetchedKeys<ValueType>(
{ attribute,
auto fetched_key = fetched_keys[fetched_key_index]; fetched_columns_index,
fetched_keys,
if (unlikely(fetched_key.is_default)) [&](auto value) { data.push_back(value); },
column_typed.insert(default_value_provider.getDefaultValue(fetched_key_index)); default_value_provider);
else
{
auto item = container[fetched_key.element_index];
data.push_back(item);
}
}
} }
}; };
@ -506,7 +494,7 @@ private:
if (unlikely(attribute.is_complex_type)) if (unlikely(attribute.is_complex_type))
{ {
auto & container = std::get<std::vector<Field>>(attribute.attribute_container); auto & container = std::get<ContainerType<Field>>(attribute.attribute_container);
std::forward<GetContainerFunc>(func)(container); std::forward<GetContainerFunc>(func)(container);
} }
else else
@ -517,7 +505,7 @@ private:
using AttributeType = typename Type::AttributeType; using AttributeType = typename Type::AttributeType;
using ValueType = DictionaryValueType<AttributeType>; using ValueType = DictionaryValueType<AttributeType>;
auto & container = std::get<PaddedPODArray<ValueType>>(attribute.attribute_container); auto & container = std::get<ContainerType<ValueType>>(attribute.attribute_container);
std::forward<GetContainerFunc>(func)(container); std::forward<GetContainerFunc>(func)(container);
}; };
@ -541,6 +529,81 @@ private:
return updated_value; return updated_value;
} }
template <typename ValueType>
static constexpr bool is_nullable_or_array_type = std::is_same_v<ValueType, Field> || std::is_same_v<ValueType, Array>;
template<typename ValueType>
using ContainerType = std::conditional_t<is_nullable_or_array_type<ValueType>, std::vector<ValueType>, PaddedPODArray<ValueType>>;
struct Attribute
{
AttributeUnderlyingType type;
bool is_complex_type;
std::variant<
ContainerType<UInt8>,
ContainerType<UInt16>,
ContainerType<UInt32>,
ContainerType<UInt64>,
ContainerType<UInt128>,
ContainerType<UInt256>,
ContainerType<Int8>,
ContainerType<Int16>,
ContainerType<Int32>,
ContainerType<Int64>,
ContainerType<Int128>,
ContainerType<Int256>,
ContainerType<Decimal32>,
ContainerType<Decimal64>,
ContainerType<Decimal128>,
ContainerType<Decimal256>,
ContainerType<Float32>,
ContainerType<Float64>,
ContainerType<UUID>,
ContainerType<StringRef>,
ContainerType<Array>,
ContainerType<Field>> attribute_container;
};
template <typename ValueType, typename ValueSetter>
void getItemsForFetchedKeys(
Attribute & attribute,
size_t fetched_keys_size,
PaddedPODArray<FetchedKey> & fetched_keys,
ValueSetter && value_setter,
const DefaultValueProvider & default_value_provider)
{
auto & container = std::get<ContainerType<ValueType>>(attribute.attribute_container);
for (size_t fetched_key_index = 0; fetched_key_index < fetched_keys_size; ++fetched_key_index)
{
auto fetched_key = fetched_keys[fetched_key_index];
if (unlikely(fetched_key.is_default))
{
auto default_value = default_value_provider.getDefaultValue(fetched_key_index);
if constexpr (std::is_same_v<ValueType, Field>)
{
value_setter(default_value);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
{
auto & value = default_value.get<String>();
value_setter(value);
}
else
{
value_setter(default_value.get<ValueType>());
}
}
else
{
value_setter(container[fetched_key.element_index]);
}
}
}
void setup(const DictionaryStructure & dictionary_structure) void setup(const DictionaryStructure & dictionary_structure)
{ {
/// For each dictionary attribute create storage attribute /// For each dictionary attribute create storage attribute
@ -563,9 +626,9 @@ private:
last_attribute.type = attribute_type; last_attribute.type = attribute_type;
if (dictionary_attribute.is_nullable) if (dictionary_attribute.is_nullable)
last_attribute.attribute_container = std::vector<Field>(); last_attribute.attribute_container = ContainerType<Field>();
else else
last_attribute.attribute_container = PaddedPODArray<ValueType>(); last_attribute.attribute_container = ContainerType<ValueType>();
}; };
callOnDictionaryAttributeType(attribute_type, type_call); callOnDictionaryAttributeType(attribute_type, type_call);
@ -582,37 +645,6 @@ private:
time_t deadline; time_t deadline;
}; };
struct Attribute
{
AttributeUnderlyingType type;
bool is_complex_type;
std::variant<
PaddedPODArray<UInt8>,
PaddedPODArray<UInt16>,
PaddedPODArray<UInt32>,
PaddedPODArray<UInt64>,
PaddedPODArray<UInt128>,
PaddedPODArray<UInt256>,
PaddedPODArray<Int8>,
PaddedPODArray<Int16>,
PaddedPODArray<Int32>,
PaddedPODArray<Int64>,
PaddedPODArray<Int128>,
PaddedPODArray<Int256>,
PaddedPODArray<Decimal32>,
PaddedPODArray<Decimal64>,
PaddedPODArray<Decimal128>,
PaddedPODArray<Decimal256>,
PaddedPODArray<Float32>,
PaddedPODArray<Float64>,
PaddedPODArray<UUID>,
PaddedPODArray<StringRef>,
/// TODO: FIX
PaddedPODArray<Array>,
std::vector<Field>> attribute_container;
};
CacheDictionaryStorageConfiguration configuration; CacheDictionaryStorageConfiguration configuration;
pcg64 rnd_engine; pcg64 rnd_engine;

View File

@ -96,7 +96,20 @@ ColumnPtr HashedDictionary<dictionary_key_type, sparse>::getColumn(
auto column = ColumnProvider::getColumn(dictionary_attribute, size); auto column = ColumnProvider::getColumn(dictionary_attribute, size);
if constexpr (std::is_same_v<ValueType, StringRef>) if constexpr (std::is_same_v<ValueType, Array>)
{
auto * out = column.get();
getItemsImpl<ValueType>(
attribute,
extractor,
[&](const size_t, const Array & value) { out->insert(value); },
[&](const size_t)
{
},
default_value_extractor);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
{ {
auto * out = column.get(); auto * out = column.get();

View File

@ -238,11 +238,21 @@ ColumnPtr IPAddressDictionary::getColumn(
auto column = ColumnProvider::getColumn(dictionary_attribute, size); auto column = ColumnProvider::getColumn(dictionary_attribute, size);
if constexpr (std::is_same_v<AttributeType, String>) if constexpr (std::is_same_v<ValueType, Array>)
{ {
auto * out = column.get(); auto * out = column.get();
getItemsImpl<ValueType, ValueType>( getItemsImpl<ValueType>(
attribute,
key_columns,
[&](const size_t, const Array & value) { out->insert(value); },
default_value_extractor);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
{
auto * out = column.get();
getItemsImpl<ValueType>(
attribute, attribute,
key_columns, key_columns,
[&](const size_t, const StringRef value) { out->insertData(value.data, value.size); }, [&](const size_t, const StringRef value) { out->insertData(value.data, value.size); },
@ -252,7 +262,7 @@ ColumnPtr IPAddressDictionary::getColumn(
{ {
auto & out = column->getData(); auto & out = column->getData();
getItemsImpl<ValueType, ValueType>( getItemsImpl<ValueType>(
attribute, attribute,
key_columns, key_columns,
[&](const size_t row, const auto value) { return out[row] = value; }, [&](const size_t row, const auto value) { return out[row] = value; },
@ -591,7 +601,7 @@ const uint8_t * IPAddressDictionary::getIPv6FromOffset(const IPAddressDictionary
return reinterpret_cast<const uint8_t *>(&ipv6_col[i * IPV6_BINARY_LENGTH]); return reinterpret_cast<const uint8_t *>(&ipv6_col[i * IPV6_BINARY_LENGTH]);
} }
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueSetter, typename DefaultValueExtractor>
void IPAddressDictionary::getItemsByTwoKeyColumnsImpl( void IPAddressDictionary::getItemsByTwoKeyColumnsImpl(
const Attribute & attribute, const Attribute & attribute,
const Columns & key_columns, const Columns & key_columns,
@ -630,7 +640,7 @@ void IPAddressDictionary::getItemsByTwoKeyColumnsImpl(
(*ipv4_col)[*found_it] == addr && (*ipv4_col)[*found_it] == addr &&
mask_column[*found_it] == mask)) mask_column[*found_it] == mask))
{ {
set_value(i, static_cast<OutputType>(vec[row_idx[*found_it]])); set_value(i, vec[row_idx[*found_it]]);
} }
else else
set_value(i, default_value_extractor[i]); set_value(i, default_value_extractor[i]);
@ -666,13 +676,13 @@ void IPAddressDictionary::getItemsByTwoKeyColumnsImpl(
if (likely(found_it != range.end() && if (likely(found_it != range.end() &&
memequal16(getIPv6FromOffset(*ipv6_col, *found_it), target.addr) && memequal16(getIPv6FromOffset(*ipv6_col, *found_it), target.addr) &&
mask_column[*found_it] == mask)) mask_column[*found_it] == mask))
set_value(i, static_cast<OutputType>(vec[row_idx[*found_it]])); set_value(i, vec[row_idx[*found_it]]);
else else
set_value(i, default_value_extractor[i]); set_value(i, default_value_extractor[i]);
} }
} }
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueSetter, typename DefaultValueExtractor>
void IPAddressDictionary::getItemsImpl( void IPAddressDictionary::getItemsImpl(
const Attribute & attribute, const Attribute & attribute,
const Columns & key_columns, const Columns & key_columns,
@ -685,7 +695,7 @@ void IPAddressDictionary::getItemsImpl(
// special case for getBlockInputStream // special case for getBlockInputStream
if (unlikely(key_columns.size() == 2)) if (unlikely(key_columns.size() == 2))
{ {
getItemsByTwoKeyColumnsImpl<AttributeType, OutputType>( getItemsByTwoKeyColumnsImpl<AttributeType>(
attribute, key_columns, std::forward<ValueSetter>(set_value), default_value_extractor); attribute, key_columns, std::forward<ValueSetter>(set_value), default_value_extractor);
query_count.fetch_add(rows, std::memory_order_relaxed); query_count.fetch_add(rows, std::memory_order_relaxed);
return; return;
@ -705,7 +715,7 @@ void IPAddressDictionary::getItemsImpl(
auto found = tryLookupIPv4(addrv4, addrv6_buf); auto found = tryLookupIPv4(addrv4, addrv6_buf);
if (found != ipNotFound()) if (found != ipNotFound())
{ {
set_value(i, static_cast<OutputType>(vec[*found])); set_value(i, vec[*found]);
++keys_found; ++keys_found;
} }
else else
@ -723,7 +733,7 @@ void IPAddressDictionary::getItemsImpl(
auto found = tryLookupIPv6(reinterpret_cast<const uint8_t *>(addr.data)); auto found = tryLookupIPv6(reinterpret_cast<const uint8_t *>(addr.data));
if (found != ipNotFound()) if (found != ipNotFound())
{ {
set_value(i, static_cast<OutputType>(vec[*found])); set_value(i, vec[*found]);
++keys_found; ++keys_found;
} }
else else

View File

@ -159,14 +159,14 @@ private:
static Attribute createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value); static Attribute createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value);
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueSetter, typename DefaultValueExtractor>
void getItemsByTwoKeyColumnsImpl( void getItemsByTwoKeyColumnsImpl(
const Attribute & attribute, const Attribute & attribute,
const Columns & key_columns, const Columns & key_columns,
ValueSetter && set_value, ValueSetter && set_value,
DefaultValueExtractor & default_value_extractor) const; DefaultValueExtractor & default_value_extractor) const;
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType,typename ValueSetter, typename DefaultValueExtractor>
void getItemsImpl( void getItemsImpl(
const Attribute & attribute, const Attribute & attribute,
const Columns & key_columns, const Columns & key_columns,

View File

@ -60,28 +60,13 @@ ColumnPtr IPolygonDictionary::getColumn(
auto result = attribute_values_column->cloneEmpty(); auto result = attribute_values_column->cloneEmpty();
result->reserve(requested_key_points.size()); result->reserve(requested_key_points.size());
Field row_value_to_insert;
size_t polygon_index = 0;
size_t keys_found = 0;
if (unlikely(attribute.is_nullable)) if (unlikely(attribute.is_nullable))
{ {
for (size_t requested_key_index = 0; requested_key_index < requested_key_points.size(); ++requested_key_index) getItemsImpl<Field>(
{ requested_key_points,
const auto found = find(requested_key_points[requested_key_index], polygon_index); [&](size_t row) { return (*attribute_values_column)[row]; },
[&](Field & value) { result->insert(value); },
if (found) default_value_provider);
{
size_t attribute_values_index = polygon_index_to_attribute_value_index[polygon_index];
attribute_values_column->get(attribute_values_index, row_value_to_insert);
++keys_found;
}
else
row_value_to_insert = default_value_provider.getDefaultValue(requested_key_index);
result->insert(row_value_to_insert);
}
} }
else else
{ {
@ -90,10 +75,8 @@ ColumnPtr IPolygonDictionary::getColumn(
using Type = std::decay_t<decltype(dictionary_attribute_type)>; using Type = std::decay_t<decltype(dictionary_attribute_type)>;
using AttributeType = typename Type::AttributeType; using AttributeType = typename Type::AttributeType;
using ValueType = DictionaryValueType<AttributeType>; using ValueType = DictionaryValueType<AttributeType>;
using ColumnType = std::conditional_t< using ColumnProvider = DictionaryAttributeColumnProvider<AttributeType>;
std::is_same_v<AttributeType, String>, using ColumnType = typename ColumnProvider::ColumnType;
ColumnString,
std::conditional_t<IsDecimalNumber<AttributeType>, ColumnDecimal<ValueType>, ColumnVector<AttributeType>>>;
const auto attribute_values_column_typed = typeid_cast<const ColumnType *>(attribute_values_column.get()); const auto attribute_values_column_typed = typeid_cast<const ColumnType *>(attribute_values_column.get());
if (!attribute_values_column_typed) if (!attribute_values_column_typed)
@ -101,54 +84,38 @@ ColumnPtr IPolygonDictionary::getColumn(
ColumnType & result_column_typed = static_cast<ColumnType &>(*result); ColumnType & result_column_typed = static_cast<ColumnType &>(*result);
if constexpr (std::is_same_v<ColumnType, ColumnString>) if constexpr (std::is_same_v<ValueType, Array>)
{ {
for (size_t requested_key_index = 0; requested_key_index < requested_key_points.size(); ++requested_key_index) getItemsImpl<ValueType>(
{ requested_key_points,
const auto found = find(requested_key_points[requested_key_index], polygon_index); [&](size_t row) { return (*attribute_values_column)[row].get<Array>(); },
[&](Array & value) { result_column_typed.insert(value); },
if (found) default_value_provider);
{
size_t attribute_values_index = polygon_index_to_attribute_value_index[polygon_index];
auto data_to_insert = attribute_values_column->getDataAt(attribute_values_index);
result_column_typed.insertData(data_to_insert.data, data_to_insert.size);
++keys_found;
}
else
result_column_typed.insert(default_value_provider.getDefaultValue(requested_key_index));
} }
else if constexpr (std::is_same_v<ValueType, StringRef>)
{
getItemsImpl<ValueType>(
requested_key_points,
[&](size_t row) { return attribute_values_column->getDataAt(row); },
[&](StringRef value) { result_column_typed.insertData(value.data, value.size); },
default_value_provider);
} }
else else
{ {
auto & attribute_data = attribute_values_column_typed->getData(); auto & attribute_data = attribute_values_column_typed->getData();
auto & result_data = result_column_typed.getData(); auto & result_data = result_column_typed.getData();
for (size_t requested_key_index = 0; requested_key_index < requested_key_points.size(); ++requested_key_index) getItemsImpl<ValueType>(
{ requested_key_points,
const auto found = find(requested_key_points[requested_key_index], polygon_index); [&](size_t row) { return attribute_data[row]; },
[&](auto value) { result_data.emplace_back(value); },
if (found) default_value_provider);
{
size_t attribute_values_index = polygon_index_to_attribute_value_index[polygon_index];
auto & item = attribute_data[attribute_values_index];
result_data.emplace_back(item);
++keys_found;
}
else
{
row_value_to_insert = default_value_provider.getDefaultValue(requested_key_index);
result_data.emplace_back(row_value_to_insert.template get<NearestFieldType<ValueType>>());
}
}
} }
}; };
callOnDictionaryAttributeType(attribute.underlying_type, type_call); callOnDictionaryAttributeType(attribute.underlying_type, type_call);
} }
query_count.fetch_add(requested_key_points.size(), std::memory_order_relaxed);
found_count.fetch_add(keys_found, std::memory_order_relaxed);
return result; return result;
} }
@ -308,6 +275,55 @@ ColumnUInt8::Ptr IPolygonDictionary::hasKeys(const Columns & key_columns, const
return result; return result;
} }
template <typename AttributeType, typename ValueGetter, typename ValueSetter, typename DefaultValueExtractor>
void IPolygonDictionary::getItemsImpl(
const std::vector<IPolygonDictionary::Point> & requested_key_points,
ValueGetter && get_value,
ValueSetter && set_value,
DefaultValueExtractor & default_value_extractor) const
{
size_t polygon_index = 0;
size_t keys_found = 0;
for (size_t requested_key_index = 0; requested_key_index < requested_key_points.size(); ++requested_key_index)
{
const auto found = find(requested_key_points[requested_key_index], polygon_index);
if (found)
{
size_t attribute_values_index = polygon_index_to_attribute_value_index[polygon_index];
auto value = get_value(attribute_values_index);
set_value(value);
++keys_found;
}
else
{
Field default_value = default_value_extractor.getDefaultValue(requested_key_index);
if constexpr (std::is_same_v<AttributeType, Field>)
{
set_value(default_value);
}
else if constexpr (std::is_same_v<AttributeType, Array>)
{
set_value(default_value.get<Array>());
}
else if constexpr (std::is_same_v<AttributeType, StringRef>)
{
auto default_value_string = default_value.get<String>();
set_value(default_value_string);
}
else
{
set_value(default_value.get<NearestFieldType<AttributeType>>());
}
}
}
query_count.fetch_add(requested_key_points.size(), std::memory_order_relaxed);
found_count.fetch_add(keys_found, std::memory_order_relaxed);
}
namespace namespace
{ {

View File

@ -139,10 +139,10 @@ private:
size_t getAttributeIndex(const std::string & attribute_name) const; size_t getAttributeIndex(const std::string & attribute_name) const;
/** Helper function for retrieving the value of an attribute by key. */ /** Helper function for retrieving the value of an attribute by key. */
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueGetter, typename ValueSetter, typename DefaultValueExtractor>
void getItemsImpl( void getItemsImpl(
size_t attribute_ind, const std::vector<IPolygonDictionary::Point> & requested_key_points,
const Columns & key_columns, ValueGetter && get_value,
ValueSetter && set_value, ValueSetter && set_value,
DefaultValueExtractor & default_value_extractor) const; DefaultValueExtractor & default_value_extractor) const;

View File

@ -131,11 +131,24 @@ ColumnPtr RangeHashedDictionary::getColumn(
auto column = ColumnProvider::getColumn(dictionary_attribute, keys_size); auto column = ColumnProvider::getColumn(dictionary_attribute, keys_size);
if constexpr (std::is_same_v<AttributeType, String>) if constexpr (std::is_same_v<ValueType, Array>)
{ {
auto * out = column.get(); auto * out = column.get();
getItemsImpl<ValueType, ValueType>( getItemsImpl<ValueType>(
attribute,
modified_key_columns,
[&](const size_t, const Array & value, bool)
{
out->insert(value);
},
default_value_extractor);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
{
auto * out = column.get();
getItemsImpl<ValueType>(
attribute, attribute,
modified_key_columns, modified_key_columns,
[&](const size_t row, const StringRef value, bool is_null) [&](const size_t row, const StringRef value, bool is_null)
@ -151,7 +164,7 @@ ColumnPtr RangeHashedDictionary::getColumn(
{ {
auto & out = column->getData(); auto & out = column->getData();
getItemsImpl<ValueType, ValueType>( getItemsImpl<ValueType>(
attribute, attribute,
modified_key_columns, modified_key_columns,
[&](const size_t row, const auto value, bool is_null) [&](const size_t row, const auto value, bool is_null)
@ -385,7 +398,7 @@ RangeHashedDictionary::createAttribute(const DictionaryAttribute& attribute, con
return attr; return attr;
} }
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueSetter, typename DefaultValueExtractor>
void RangeHashedDictionary::getItemsImpl( void RangeHashedDictionary::getItemsImpl(
const Attribute & attribute, const Attribute & attribute,
const Columns & key_columns, const Columns & key_columns,
@ -423,7 +436,7 @@ void RangeHashedDictionary::getItemsImpl(
auto & value = val_it->value; auto & value = val_it->value;
if (value) if (value)
set_value(row, static_cast<OutputType>(*value), false); // NOLINT set_value(row, *value, false);
else else
set_value(row, default_value_extractor[row], true); set_value(row, default_value_extractor[row], true);
} }

View File

@ -170,7 +170,7 @@ private:
static Attribute createAttribute(const DictionaryAttribute& attribute, const Field & null_value); static Attribute createAttribute(const DictionaryAttribute& attribute, const Field & null_value);
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor> template <typename AttributeType, typename ValueSetter, typename DefaultValueExtractor>
void getItemsImpl( void getItemsImpl(
const Attribute & attribute, const Attribute & attribute,
const Columns & key_columns, const Columns & key_columns,

View File

@ -17,6 +17,7 @@ bool ParserDictionaryAttributeDeclaration::parseImpl(Pos & pos, ASTPtr & node, E
ParserKeyword s_injective{"INJECTIVE"}; ParserKeyword s_injective{"INJECTIVE"};
ParserKeyword s_is_object_id{"IS_OBJECT_ID"}; ParserKeyword s_is_object_id{"IS_OBJECT_ID"};
ParserLiteral default_parser; ParserLiteral default_parser;
ParserArrayOfLiterals array_literals_parser;
ParserTernaryOperatorExpression expression_parser; ParserTernaryOperatorExpression expression_parser;
/// mandatory attribute name /// mandatory attribute name
@ -40,8 +41,10 @@ bool ParserDictionaryAttributeDeclaration::parseImpl(Pos & pos, ASTPtr & node, E
{ {
if (!default_value && s_default.ignore(pos, expected)) if (!default_value && s_default.ignore(pos, expected))
{ {
if (!default_parser.parse(pos, default_value, expected)) if (!default_parser.parse(pos, default_value, expected) &&
!array_literals_parser.parse(pos, default_value, expected))
return false; return false;
continue; continue;
} }

View File

@ -1,2 +0,0 @@
Flat dictionary
[0,1,2]

View File

@ -1,55 +0,0 @@
DROP TABLE IF EXISTS dictionary_array_source_table;
CREATE TABLE dictionary_array_source_table
(
id UInt64,
array_value Array(Int) DEFAULT [0, 1, 3]
) ENGINE=TinyLog;
INSERT INTO dictionary_array_source_table VALUES (0, [0, 1, 2]);
DROP DICTIONARY IF EXISTS flat_dictionary;
CREATE DICTIONARY flat_dictionary
(
id UInt64,
array_value Array(Int)
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'dictionary_array_source_table'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(FLAT());
SELECT 'Flat dictionary';
SELECT dictGet('flat_dictionary', 'array_value', toUInt64(0));
DROP DICTIONARY flat_dictionary;
-- DROP DICTIONARY IF EXISTS hashed_dictionary;
-- CREATE DICTIONARY hashed_dictionary
-- (
-- id UInt64,
-- array_value Array(Int)
-- )
-- PRIMARY KEY id
-- SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'dictionary_array_source_table'))
-- LIFETIME(MIN 1 MAX 1000)
-- LAYOUT(HASHED());
-- SELECT 'Hashed dictionary';
-- SELECT dictGet('hashed_dictionary', 'array_value', toUInt64(1));
-- DROP DICTIONARY hashed_dictionary;
-- DROP DICTIONARY IF EXISTS cache_dictionary;
-- CREATE DICTIONARY cache_dictionary
-- (
-- id UInt64,
-- array_value Array(Int)
-- )
-- PRIMARY KEY id
-- SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'dictionary_array_source_table'))
-- LIFETIME(MIN 1 MAX 1000)
-- LAYOUT(CACHE(SIZE_IN_CELLS 10));
-- SELECT 'Cache dictionary';
-- SELECT dictGet('cache_dictionary', 'array_value', toUInt64(1));
-- DROP DICTIONARY cache_dictionary;
DROP TABLE dictionary_array_source_table;

View File

@ -0,0 +1,28 @@
Flat dictionary
[0,1,2]
[1,2,3]
[2,3,4]
Hashed dictionary
[0,1,2]
[1,2,3]
[2,3,4]
Cache dictionary
[0,1,2]
[1,2,3]
[2,3,4]
Direct dictionary
[0,1,2]
[1,2,3]
[2,3,4]
IPTrie dictionary
[0,1,2]
[1,2,3]
[2,3,4]
Polygon dictionary
[0,1,2]
[1,2,3]
[2,3,4]
Range dictionary
[0,1,2]
[1,2,3]
[2,3,4]

View File

@ -0,0 +1,164 @@
DROP TABLE IF EXISTS dictionary_array_source_table;
CREATE TABLE dictionary_array_source_table
(
id UInt64,
array_value Array(Int64)
) ENGINE=TinyLog;
INSERT INTO dictionary_array_source_table VALUES (0, [0, 1, 2]);
DROP DICTIONARY IF EXISTS flat_dictionary;
CREATE DICTIONARY flat_dictionary
(
id UInt64,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'dictionary_array_source_table'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(FLAT());
SELECT 'Flat dictionary';
SELECT dictGet('flat_dictionary', 'array_value', toUInt64(0));
SELECT dictGet('flat_dictionary', 'array_value', toUInt64(1));
SELECT dictGetOrDefault('flat_dictionary', 'array_value', toUInt64(1), [2,3,4]);
DROP DICTIONARY flat_dictionary;
DROP DICTIONARY IF EXISTS hashed_dictionary;
CREATE DICTIONARY hashed_dictionary
(
id UInt64,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'dictionary_array_source_table'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(HASHED());
SELECT 'Hashed dictionary';
SELECT dictGet('hashed_dictionary', 'array_value', toUInt64(0));
SELECT dictGet('hashed_dictionary', 'array_value', toUInt64(1));
SELECT dictGetOrDefault('hashed_dictionary', 'array_value', toUInt64(1), [2,3,4]);
DROP DICTIONARY hashed_dictionary;
DROP DICTIONARY IF EXISTS cache_dictionary;
CREATE DICTIONARY cache_dictionary
(
id UInt64,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'dictionary_array_source_table'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(CACHE(SIZE_IN_CELLS 10));
SELECT 'Cache dictionary';
SELECT dictGet('cache_dictionary', 'array_value', toUInt64(0));
SELECT dictGet('cache_dictionary', 'array_value', toUInt64(1));
SELECT dictGetOrDefault('cache_dictionary', 'array_value', toUInt64(1), [2,3,4]);
DROP DICTIONARY cache_dictionary;
DROP DICTIONARY IF EXISTS direct_dictionary;
CREATE DICTIONARY direct_dictionary
(
id UInt64,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'dictionary_array_source_table'))
LAYOUT(DIRECT());
SELECT 'Direct dictionary';
SELECT dictGet('direct_dictionary', 'array_value', toUInt64(0));
SELECT dictGet('direct_dictionary', 'array_value', toUInt64(1));
SELECT dictGetOrDefault('direct_dictionary', 'array_value', toUInt64(1), [2,3,4]);
DROP DICTIONARY direct_dictionary;
DROP TABLE IF EXISTS ip_trie_dictionary_array_source_table;
CREATE TABLE ip_trie_dictionary_array_source_table
(
prefix String,
array_value Array(Int64)
) ENGINE = TinyLog;
DROP TABLE dictionary_array_source_table;
DROP DICTIONARY IF EXISTS ip_trie_dictionary;
CREATE DICTIONARY ip_trie_dictionary
(
prefix String,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY prefix
SOURCE(CLICKHOUSE(HOST 'localhost' port tcpPort() TABLE 'ip_trie_dictionary_array_source_table'))
LIFETIME(MIN 10 MAX 1000)
LAYOUT(IP_TRIE());
INSERT INTO ip_trie_dictionary_array_source_table VALUES ('127.0.0.0', [0, 1, 2]);
SELECT 'IPTrie dictionary';
SELECT dictGet('ip_trie_dictionary', 'array_value', tuple(IPv4StringToNum('127.0.0.0')));
SELECT dictGet('ip_trie_dictionary', 'array_value', tuple(IPv4StringToNum('128.0.0.0')));
SELECT dictGetOrDefault('ip_trie_dictionary', 'array_value', tuple(IPv4StringToNum('128.0.0.0')), [2,3,4]);
DROP DICTIONARY ip_trie_dictionary;
DROP TABLE ip_trie_dictionary_array_source_table;
DROP TABLE IF EXISTS polygon_dictionary_array_source_table;
CREATE TABLE polygon_dictionary_array_source_table
(
key Array(Array(Array(Tuple(Float64, Float64)))),
array_value Array(Int64)
) ENGINE = TinyLog;
INSERT INTO polygon_dictionary_array_source_table VALUES ([[[(0, 0), (0, 1), (1, 1), (1, 0)]]], [0, 1, 2]);
DROP DICTIONARY IF EXISTS polygon_dictionary;
CREATE DICTIONARY polygon_dictionary
(
key Array(Array(Array(Tuple(Float64, Float64)))),
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY key
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'polygon_dictionary_array_source_table'))
LIFETIME(MIN 0 MAX 1000)
LAYOUT(POLYGON());
SELECT 'Polygon dictionary';
SELECT dictGet('polygon_dictionary', 'array_value', tuple(0.5, 0.5));
SELECT dictGet('polygon_dictionary', 'array_value', tuple(1.5, 1.5));
SELECT dictGetOrDefault('polygon_dictionary', 'array_value', tuple(1.5, 1.5), [2, 3, 4]);
DROP DICTIONARY polygon_dictionary;
DROP TABLE polygon_dictionary_array_source_table;
CREATE TABLE range_dictionary_array_source_table
(
key UInt64,
start_date Date,
end_date Date,
array_value Array(Int64)
)
ENGINE = TinyLog;
INSERT INTO range_dictionary_array_source_table VALUES(1, toDate('2019-05-05'), toDate('2019-05-20'), [0, 1, 2]);
CREATE DICTIONARY range_dictionary
(
key UInt64,
start_date Date,
end_date Date,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY key
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'range_dictionary_array_source_table'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(RANGE_HASHED())
RANGE(MIN start_date MAX end_date);
SELECT 'Range dictionary';
SELECT dictGet('range_dictionary', 'array_value', toUInt64(1), toDate('2019-05-15'));
SELECT dictGet('range_dictionary', 'array_value', toUInt64(1), toDate('2019-05-21'));
SELECT dictGetOrDefault('range_dictionary', 'array_value', toUInt64(1), toDate('2019-05-21'), [2, 3, 4]);
DROP DICTIONARY range_dictionary;
DROP TABLE range_dictionary_array_source_table;

View File

@ -0,0 +1,4 @@
SSDCache dictionary
[0,1,2]
[1,2,3]
[2,3,4]

View File

@ -0,0 +1,35 @@
#!/usr/bin/env bash
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
USER_FILES_PATH=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}')
$CLICKHOUSE_CLIENT -n --query="
DROP TABLE IF EXISTS dictionary_array_source_table;
CREATE TABLE dictionary_array_source_table
(
id UInt64,
array_value Array(Int64)
) ENGINE=TinyLog;
INSERT INTO dictionary_array_source_table VALUES (0, [0, 1, 2]);
DROP DICTIONARY IF EXISTS ssd_cache_dictionary;
CREATE DICTIONARY ssd_cache_dictionary
(
id UInt64,
array_value Array(Int64) DEFAULT [1,2,3]
)
PRIMARY KEY id
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'dictionary_array_source_table'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 8192 PATH '$USER_FILES_PATH/0d'));
SELECT 'SSDCache dictionary';
SELECT dictGet('ssd_cache_dictionary', 'array_value', toUInt64(0));
SELECT dictGet('ssd_cache_dictionary', 'array_value', toUInt64(1));
SELECT dictGetOrDefault('ssd_cache_dictionary', 'array_value', toUInt64(1), [2,3,4]);
DROP DICTIONARY ssd_cache_dictionary;
DROP TABLE dictionary_array_source_table;"