thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #63723 from Algunenano/ifunction_column_size

Browse Source 
Add a bunch of important asserts
This commit is contained in:
Raúl Marín 2024-06-03 14:04:02 +00:00 committed by GitHub
commit ee416a6862
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
31 changed files with 218 additions and 417 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
programs/keeper/CMakeLists.txt
View File

@ -9,8 +9,6 @@ set (CLICKHOUSE_KEEPER_LINK
clickhouse_common_zookeeper
daemon
dbms
${LINK_RESOURCE_LIB}
)
clickhouse_program_add(keeper)
@ -210,8 +208,6 @@ if (BUILD_STANDALONE_KEEPER)
loggers_no_text_log
clickhouse_common_io
clickhouse_parsers # Otherwise compression will not built. FIXME.
${LINK_RESOURCE_LIB_STANDALONE_KEEPER}
)
set_target_properties(clickhouse-keeper PROPERTIES RUNTIME_OUTPUT_DIRECTORY ../)

2
programs/server/CMakeLists.txt
View File

@ -14,8 +14,6 @@ set (CLICKHOUSE_SERVER_LINK
clickhouse_storages_system
clickhouse_table_functions
${LINK_RESOURCE_LIB}
PUBLIC
daemon
)

7
src/Columns/IColumnDummy.cpp
View File

@ -60,12 +60,9 @@ ColumnPtr IColumnDummy::filter(const Filter & filt, ssize_t /*result_size_hint*/
return cloneDummy(bytes);
}
void IColumnDummy::expand(const IColumn::Filter & mask, bool inverted)
void IColumnDummy::expand(const IColumn::Filter & mask, bool)
{
size_t bytes = countBytesInFilter(mask);
if (inverted)
bytes = mask.size() - bytes;
s = bytes;
s = mask.size();
}
ColumnPtr IColumnDummy::permute(const Permutation & perm, size_t limit) const

61
src/Columns/MaskOperations.cpp
View File

@ -77,7 +77,7 @@ INSTANTIATE(IPv6)
#undef INSTANTIATE
template <bool inverted, bool column_is_short, typename Container>
template <bool inverted, typename Container>
static size_t extractMaskNumericImpl(
PaddedPODArray<UInt8> & mask,
const Container & data,
@ -85,42 +85,27 @@ static size_t extractMaskNumericImpl(
const PaddedPODArray<UInt8> * null_bytemap,
PaddedPODArray<UInt8> * nulls)
{
if constexpr (!column_is_short)
{
if (data.size() != mask.size())
throw Exception(ErrorCodes::LOGICAL_ERROR, "The size of a full data column is not equal to the size of a mask");
}
if (data.size() != mask.size())
throw Exception(ErrorCodes::LOGICAL_ERROR, "The size of a full data column is not equal to the size of a mask");
size_t ones_count = 0;
size_t data_index = 0;
size_t mask_size = mask.size();
size_t data_size = data.size();
for (size_t i = 0; i != mask_size && data_index != data_size; ++i)
for (size_t i = 0; i != mask_size; ++i)
{
// Change mask only where value is 1.
if (!mask[i])
continue;
UInt8 value;
size_t index;
if constexpr (column_is_short)
{
index = data_index;
++data_index;
}
else
index = i;
if (null_bytemap && (*null_bytemap)[index])
if (null_bytemap && (*null_bytemap)[i])
{
value = null_value;
if (nulls)
(*nulls)[i] = 1;
}
else
value = static_cast<bool>(data[index]);
value = static_cast<bool>(data[i]);
if constexpr (inverted)
value = !value;
@ -131,12 +116,6 @@ static size_t extractMaskNumericImpl(
mask[i] = value;
}
if constexpr (column_is_short)
{
if (data_index != data_size)
throw Exception(ErrorCodes::LOGICAL_ERROR, "The size of a short column is not equal to the number of ones in a mask");
}
return ones_count;
}
@ -155,10 +134,7 @@ static bool extractMaskNumeric(
const auto & data = numeric_column->getData();
size_t ones_count;
if (column->size() < mask.size())
ones_count = extractMaskNumericImpl<inverted, true>(mask, data, null_value, null_bytemap, nulls);
else
ones_count = extractMaskNumericImpl<inverted, false>(mask, data, null_value, null_bytemap, nulls);
ones_count = extractMaskNumericImpl<inverted>(mask, data, null_value, null_bytemap, nulls);
mask_info.has_ones = ones_count > 0;
mask_info.has_zeros = ones_count != mask.size();
@ -279,25 +255,32 @@ void maskedExecute(ColumnWithTypeAndName & column, const PaddedPODArray<UInt8> &
if (!column_function)
return;
size_t original_size = column.column->size();
ColumnWithTypeAndName result;
/// If mask contains only zeros, we can just create
/// an empty column with the execution result type.
if (!mask_info.has_ones)
{
/// If mask contains only zeros, we can just create a column with default values as it will be ignored
auto result_type = column_function->getResultType();
auto empty_column = result_type->createColumn();
result = {std::move(empty_column), result_type, ""};
auto default_column = result_type->createColumnConstWithDefaultValue(original_size)->convertToFullColumnIfConst();
column = {default_column, result_type, ""};
}
/// Filter column only if mask contains zeros.
else if (mask_info.has_zeros)
{
/// If it contains both zeros and ones, we need to execute the function only on the mask values
/// First we filter the column, which creates a new column, then we apply the column, and finally we expand it
/// Expanding is done to keep consistency in function calls (all columns the same size) and it's ok
/// since the values won't be used by `if`
auto filtered = column_function->filter(mask, -1);
result = typeid_cast<const ColumnFunction *>(filtered.get())->reduce();
auto filter_after_execution = typeid_cast<const ColumnFunction *>(filtered.get())->reduce();
auto mut_column = IColumn::mutate(std::move(filter_after_execution.column));
mut_column->expand(mask, false);
column.column = std::move(mut_column);
}
else
result = column_function->reduce();
column = column_function->reduce();
column = std::move(result);
chassert(column.column->size() == original_size);
}
void executeColumnIfNeeded(ColumnWithTypeAndName & column, bool empty)

6
src/Dictionaries/CacheDictionary.cpp
View File

@ -450,7 +450,10 @@ MutableColumns CacheDictionary<dictionary_key_type>::aggregateColumnsInOrderOfKe
if (default_mask)
{
if (state.isDefault())
{
(*default_mask)[key_index] = 1;
aggregated_column->insertDefault();
}
else
{
(*default_mask)[key_index] = 0;
@ -536,7 +539,10 @@ MutableColumns CacheDictionary<dictionary_key_type>::aggregateColumns(
}
if (default_mask)
{
aggregated_column->insertDefault(); /// Any default is ok
(*default_mask)[key_index] = 1;
}
else
{
/// Insert default value

15
src/Dictionaries/CacheDictionaryStorage.h
View File

@ -189,7 +189,6 @@ private:
const time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
size_t fetched_columns_index = 0;
size_t fetched_columns_index_without_default = 0;
size_t keys_size = keys.size();
PaddedPODArray<FetchedKey> fetched_keys;
@ -211,15 +210,10 @@ private:
result.expired_keys_size += static_cast<size_t>(key_state == KeyState::expired);
result.key_index_to_state[key_index] = {key_state,
default_mask ? fetched_columns_index_without_default : fetched_columns_index};
result.key_index_to_state[key_index] = {key_state, fetched_columns_index};
fetched_keys[fetched_columns_index] = FetchedKey(cell.element_index, cell.is_default);
++fetched_columns_index;
if (!cell.is_default)
++fetched_columns_index_without_default;
result.key_index_to_state[key_index].setDefaultValue(cell.is_default);
result.default_keys_size += cell.is_default;
}
@ -233,8 +227,7 @@ private:
auto & attribute = attributes[attribute_index];
auto & fetched_column = *result.fetched_columns[attribute_index];
fetched_column.reserve(default_mask ? fetched_columns_index_without_default :
fetched_columns_index);
fetched_column.reserve(fetched_columns_index);
if (!default_mask)
{
@ -689,7 +682,11 @@ private:
auto fetched_key = fetched_keys[fetched_key_index];
if (unlikely(fetched_key.is_default))
{
default_mask[fetched_key_index] = 1;
auto v = ValueType{};
value_setter(v);
}
else
{
default_mask[fetched_key_index] = 0;

3
src/Dictionaries/DirectDictionary.cpp
View File

@ -174,6 +174,9 @@ Columns DirectDictionary<dictionary_key_type>::getColumns(
{
if (!mask_filled)
(*default_mask)[requested_key_index] = 1;
Field value{};
result_column->insert(value);
}
else
{

49
src/Dictionaries/FlatDictionary.cpp
View File

@ -92,24 +92,20 @@ ColumnPtr FlatDictionary::getColumn(
if (is_short_circuit)
{
IColumn::Filter & default_mask = std::get<RefFilter>(default_or_filter).get();
size_t keys_found = 0;
if constexpr (std::is_same_v<ValueType, Array>)
{
auto * out = column.get();
keys_found = getItemsShortCircuitImpl<ValueType, false>(
attribute,
ids,
[&](size_t, const Array & value, bool) { out->insert(value); },
default_mask);
getItemsShortCircuitImpl<ValueType, false>(
attribute, ids, [&](size_t, const Array & value, bool) { out->insert(value); }, default_mask);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
{
auto * out = column.get();
if (is_attribute_nullable)
keys_found = getItemsShortCircuitImpl<ValueType, true>(
getItemsShortCircuitImpl<ValueType, true>(
attribute,
ids,
[&](size_t row, StringRef value, bool is_null)
@ -119,18 +115,15 @@ ColumnPtr FlatDictionary::getColumn(
},
default_mask);
else
keys_found = getItemsShortCircuitImpl<ValueType, false>(
attribute,
ids,
[&](size_t, StringRef value, bool) { out->insertData(value.data, value.size); },
default_mask);
getItemsShortCircuitImpl<ValueType, false>(
attribute, ids, [&](size_t, StringRef value, bool) { out->insertData(value.data, value.size); }, default_mask);
}
else
{
auto & out = column->getData();
if (is_attribute_nullable)
keys_found = getItemsShortCircuitImpl<ValueType, true>(
getItemsShortCircuitImpl<ValueType, true>(
attribute,
ids,
[&](size_t row, const auto value, bool is_null)
@ -140,17 +133,9 @@ ColumnPtr FlatDictionary::getColumn(
},
default_mask);
else
keys_found = getItemsShortCircuitImpl<ValueType, false>(
attribute,
ids,
[&](size_t row, const auto value, bool) { out[row] = value; },
default_mask);
out.resize(keys_found);
getItemsShortCircuitImpl<ValueType, false>(
attribute, ids, [&](size_t row, const auto value, bool) { out[row] = value; }, default_mask);
}
if (attribute.is_nullable_set)
vec_null_map_to->resize(keys_found);
}
else
{
@ -643,11 +628,8 @@ void FlatDictionary::getItemsImpl(
}
template <typename AttributeType, bool is_nullable, typename ValueSetter>
size_t FlatDictionary::getItemsShortCircuitImpl(
const Attribute & attribute,
const PaddedPODArray<UInt64> & keys,
ValueSetter && set_value,
IColumn::Filter & default_mask) const
void FlatDictionary::getItemsShortCircuitImpl(
const Attribute & attribute, const PaddedPODArray<UInt64> & keys, ValueSetter && set_value, IColumn::Filter & default_mask) const
{
const auto rows = keys.size();
default_mask.resize(rows);
@ -660,22 +642,23 @@ size_t FlatDictionary::getItemsShortCircuitImpl(
if (key < loaded_keys.size() && loaded_keys[key])
{
keys_found++;
default_mask[row] = 0;
if constexpr (is_nullable)
set_value(keys_found, container[key], attribute.is_nullable_set->find(key) != nullptr);
set_value(row, container[key], attribute.is_nullable_set->find(key) != nullptr);
else
set_value(keys_found, container[key], false);
++keys_found;
set_value(row, container[key], false);
}
else
{
default_mask[row] = 1;
set_value(row, AttributeType{}, true);
}
}
query_count.fetch_add(rows, std::memory_order_relaxed);
found_count.fetch_add(keys_found, std::memory_order_relaxed);
return keys_found;
}
template <typename T>

7
src/Dictionaries/FlatDictionary.h
View File

@ -166,11 +166,8 @@ private:
DefaultValueExtractor & default_value_extractor) const;
template <typename AttributeType, bool is_nullable, typename ValueSetter>
size_t getItemsShortCircuitImpl(
const Attribute & attribute,
const PaddedPODArray<UInt64> & keys,
ValueSetter && set_value,
IColumn::Filter & default_mask) const;
void getItemsShortCircuitImpl(
const Attribute & attribute, const PaddedPODArray<UInt64> & keys, ValueSetter && set_value, IColumn::Filter & default_mask) const;
template <typename T>
void resize(Attribute & attribute, UInt64 key);

42
src/Dictionaries/HashedArrayDictionary.cpp
View File

@ -650,24 +650,20 @@ ColumnPtr HashedArrayDictionary<dictionary_key_type, sharded>::getAttributeColum
if (is_short_circuit)
{
IColumn::Filter & default_mask = std::get<RefFilter>(default_or_filter).get();
size_t keys_found = 0;
if constexpr (std::is_same_v<ValueType, Array>)
{
auto * out = column.get();
keys_found = getItemsShortCircuitImpl<ValueType, false>(
attribute,
keys_object,
[&](const size_t, const Array & value, bool) { out->insert(value); },
default_mask);
getItemsShortCircuitImpl<ValueType, false>(
attribute, keys_object, [&](const size_t, const Array & value, bool) { out->insert(value); }, default_mask);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
{
auto * out = column.get();
if (is_attribute_nullable)
keys_found = getItemsShortCircuitImpl<ValueType, true>(
getItemsShortCircuitImpl<ValueType, true>(
attribute,
keys_object,
[&](size_t row, StringRef value, bool is_null)
@ -677,7 +673,7 @@ ColumnPtr HashedArrayDictionary<dictionary_key_type, sharded>::getAttributeColum
},
default_mask);
else
keys_found = getItemsShortCircuitImpl<ValueType, false>(
getItemsShortCircuitImpl<ValueType, false>(
attribute,
keys_object,
[&](size_t, StringRef value, bool) { out->insertData(value.data, value.size); },
@ -688,7 +684,7 @@ ColumnPtr HashedArrayDictionary<dictionary_key_type, sharded>::getAttributeColum
auto & out = column->getData();
if (is_attribute_nullable)
keys_found = getItemsShortCircuitImpl<ValueType, true>(
getItemsShortCircuitImpl<ValueType, true>(
attribute,
keys_object,
[&](size_t row, const auto value, bool is_null)
@ -698,17 +694,9 @@ ColumnPtr HashedArrayDictionary<dictionary_key_type, sharded>::getAttributeColum
},
default_mask);
else
keys_found = getItemsShortCircuitImpl<ValueType, false>(
attribute,
keys_object,
[&](size_t row, const auto value, bool) { out[row] = value; },
default_mask);
out.resize(keys_found);
getItemsShortCircuitImpl<ValueType, false>(
attribute, keys_object, [&](size_t row, const auto value, bool) { out[row] = value; }, default_mask);
}
if (is_attribute_nullable)
vec_null_map_to->resize(keys_found);
}
else
{
@ -834,7 +822,7 @@ void HashedArrayDictionary<dictionary_key_type, sharded>::getItemsImpl(
template <DictionaryKeyType dictionary_key_type, bool sharded>
template <typename AttributeType, bool is_nullable, typename ValueSetter>
size_t HashedArrayDictionary<dictionary_key_type, sharded>::getItemsShortCircuitImpl(
void HashedArrayDictionary<dictionary_key_type, sharded>::getItemsShortCircuitImpl(
const Attribute & attribute,
DictionaryKeysExtractor<dictionary_key_type> & keys_extractor,
ValueSetter && set_value,
@ -870,14 +858,16 @@ size_t HashedArrayDictionary<dictionary_key_type, sharded>::getItemsShortCircuit
++keys_found;
}
else
{
default_mask[key_index] = 1;
set_value(key_index, AttributeType{}, true);
}
keys_extractor.rollbackCurrentKey();
}
query_count.fetch_add(keys_size, std::memory_order_relaxed);
found_count.fetch_add(keys_found, std::memory_order_relaxed);
return keys_found;
}
template <DictionaryKeyType dictionary_key_type, bool sharded>
@ -929,7 +919,7 @@ void HashedArrayDictionary<dictionary_key_type, sharded>::getItemsImpl(
template <DictionaryKeyType dictionary_key_type, bool sharded>
template <typename AttributeType, bool is_nullable, typename ValueSetter>
size_t HashedArrayDictionary<dictionary_key_type, sharded>::getItemsShortCircuitImpl(
void HashedArrayDictionary<dictionary_key_type, sharded>::getItemsShortCircuitImpl(
const Attribute & attribute,
const KeyIndexToElementIndex & key_index_to_element_index,
ValueSetter && set_value,
@ -938,7 +928,6 @@ size_t HashedArrayDictionary<dictionary_key_type, sharded>::getItemsShortCircuit
const auto & attribute_containers = std::get<AttributeContainerShardsType<AttributeType>>(attribute.containers);
const size_t keys_size = key_index_to_element_index.size();
size_t shard = 0;
size_t keys_found = 0;
for (size_t key_index = 0; key_index < keys_size; ++key_index)
{
@ -955,7 +944,6 @@ size_t HashedArrayDictionary<dictionary_key_type, sharded>::getItemsShortCircuit
if (element_index != -1)
{
keys_found++;
const auto & attribute_container = attribute_containers[shard];
size_t found_element_index = static_cast<size_t>(element_index);
@ -966,9 +954,11 @@ size_t HashedArrayDictionary<dictionary_key_type, sharded>::getItemsShortCircuit
else
set_value(key_index, element, false);
}
else
{
set_value(key_index, AttributeType{}, true);
}
}
return keys_found;
}
template <DictionaryKeyType dictionary_key_type, bool sharded>

4
src/Dictionaries/HashedArrayDictionary.h
View File

@ -228,7 +228,7 @@ private:
DefaultValueExtractor & default_value_extractor) const;
template <typename AttributeType, bool is_nullable, typename ValueSetter>
size_t getItemsShortCircuitImpl(
void getItemsShortCircuitImpl(
const Attribute & attribute,
DictionaryKeysExtractor<dictionary_key_type> & keys_extractor,
ValueSetter && set_value,
@ -244,7 +244,7 @@ private:
DefaultValueExtractor & default_value_extractor) const;
template <typename AttributeType, bool is_nullable, typename ValueSetter>
size_t getItemsShortCircuitImpl(
void getItemsShortCircuitImpl(
const Attribute & attribute,
const KeyIndexToElementIndex & key_index_to_element_index,
ValueSetter && set_value,

44
src/Dictionaries/HashedDictionary.h
View File

@ -245,12 +245,12 @@ private:
ValueSetter && set_value,
DefaultValueExtractor & default_value_extractor) const;
template <typename AttributeType, bool is_nullable, typename ValueSetter, typename NullSetter>
size_t getItemsShortCircuitImpl(
template <typename AttributeType, bool is_nullable, typename ValueSetter, typename NullAndDefaultSetter>
void getItemsShortCircuitImpl(
const Attribute & attribute,
DictionaryKeysExtractor<dictionary_key_type> & keys_extractor,
ValueSetter && set_value,
NullSetter && set_null,
NullAndDefaultSetter && set_null_and_default,
IColumn::Filter & default_mask) const;
template <typename GetContainersFunc>
@ -428,17 +428,16 @@ ColumnPtr HashedDictionary<dictionary_key_type, sparse, sharded>::getColumn(
if (is_short_circuit)
{
IColumn::Filter & default_mask = std::get<RefFilter>(default_or_filter).get();
size_t keys_found = 0;
if constexpr (std::is_same_v<ValueType, Array>)
{
auto * out = column.get();
keys_found = getItemsShortCircuitImpl<ValueType, false>(
getItemsShortCircuitImpl<ValueType, false>(
attribute,
extractor,
[&](const size_t, const Array & value) { out->insert(value); },
[&](size_t) {},
[&](size_t) { out->insertDefault(); },
default_mask);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
@ -447,7 +446,7 @@ ColumnPtr HashedDictionary<dictionary_key_type, sparse, sharded>::getColumn(
if (is_attribute_nullable)
{
keys_found = getItemsShortCircuitImpl<ValueType, true>(
getItemsShortCircuitImpl<ValueType, true>(
attribute,
extractor,
[&](size_t row, StringRef value)
@ -463,11 +462,11 @@ ColumnPtr HashedDictionary<dictionary_key_type, sparse, sharded>::getColumn(
default_mask);
}
else
keys_found = getItemsShortCircuitImpl<ValueType, false>(
getItemsShortCircuitImpl<ValueType, false>(
attribute,
extractor,
[&](size_t, StringRef value) { out->insertData(value.data, value.size); },
[&](size_t) {},
[&](size_t) { out->insertDefault(); },
default_mask);
}
else
@ -475,7 +474,7 @@ ColumnPtr HashedDictionary<dictionary_key_type, sparse, sharded>::getColumn(
auto & out = column->getData();
if (is_attribute_nullable)
keys_found = getItemsShortCircuitImpl<ValueType, true>(
getItemsShortCircuitImpl<ValueType, true>(
attribute,
extractor,
[&](size_t row, const auto value)
@ -486,18 +485,9 @@ ColumnPtr HashedDictionary<dictionary_key_type, sparse, sharded>::getColumn(
[&](size_t row) { (*vec_null_map_to)[row] = true; },
default_mask);
else
keys_found = getItemsShortCircuitImpl<ValueType, false>(
attribute,
extractor,
[&](size_t row, const auto value) { out[row] = value; },
[&](size_t) {},
default_mask);
out.resize(keys_found);
getItemsShortCircuitImpl<ValueType, false>(
attribute, extractor, [&](size_t row, const auto value) { out[row] = value; }, [&](size_t) {}, default_mask);
}
if (is_attribute_nullable)
vec_null_map_to->resize(keys_found);
}
else
{
@ -1112,12 +1102,12 @@ void HashedDictionary<dictionary_key_type, sparse, sharded>::getItemsImpl(
}
template <DictionaryKeyType dictionary_key_type, bool sparse, bool sharded>
template <typename AttributeType, bool is_nullable, typename ValueSetter, typename NullSetter>
size_t HashedDictionary<dictionary_key_type, sparse, sharded>::getItemsShortCircuitImpl(
template <typename AttributeType, bool is_nullable, typename ValueSetter, typename NullAndDefaultSetter>
void HashedDictionary<dictionary_key_type, sparse, sharded>::getItemsShortCircuitImpl(
const Attribute & attribute,
DictionaryKeysExtractor<dictionary_key_type> & keys_extractor,
ValueSetter && set_value,
NullSetter && set_null,
NullAndDefaultSetter && set_null_and_default,
IColumn::Filter & default_mask) const
{
const auto & attribute_containers = std::get<CollectionsHolder<AttributeType>>(attribute.containers);
@ -1143,20 +1133,22 @@ size_t HashedDictionary<dictionary_key_type, sparse, sharded>::getItemsShortCirc
// Need to consider items in is_nullable_sets as well, see blockToAttributes()
else if (is_nullable && (*attribute.is_nullable_sets)[shard].find(key) != nullptr)
{
set_null(key_index);
set_null_and_default(key_index);
default_mask[key_index] = 0;
++keys_found;
}
else
{
set_null_and_default(key_index);
default_mask[key_index] = 1;
}
keys_extractor.rollbackCurrentKey();
}
query_count.fetch_add(keys_size, std::memory_order_relaxed);
found_count.fetch_add(keys_found, std::memory_order_relaxed);
return keys_found;
}
template <DictionaryKeyType dictionary_key_type, bool sparse, bool sharded>

47
src/Dictionaries/IPAddressDictionary.cpp
View File

@ -249,39 +249,27 @@ ColumnPtr IPAddressDictionary::getColumn(
if (is_short_circuit)
{
IColumn::Filter & default_mask = std::get<RefFilter>(default_or_filter).get();
size_t keys_found = 0;
if constexpr (std::is_same_v<ValueType, Array>)
{
auto * out = column.get();
keys_found = getItemsShortCircuitImpl<ValueType>(
attribute,
key_columns,
[&](const size_t, const Array & value) { out->insert(value); },
default_mask);
getItemsShortCircuitImpl<ValueType>(
attribute, key_columns, [&](const size_t, const Array & value) { out->insert(value); }, default_mask);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
{
auto * out = column.get();
keys_found = getItemsShortCircuitImpl<ValueType>(
attribute,
key_columns,
[&](const size_t, StringRef value) { out->insertData(value.data, value.size); },
default_mask);
getItemsShortCircuitImpl<ValueType>(
attribute, key_columns, [&](const size_t, StringRef value) { out->insertData(value.data, value.size); }, default_mask);
}
else
{
auto & out = column->getData();
keys_found = getItemsShortCircuitImpl<ValueType>(
attribute,
key_columns,
[&](const size_t row, const auto value) { return out[row] = value; },
default_mask);
out.resize(keys_found);
getItemsShortCircuitImpl<ValueType>(
attribute, key_columns, [&](const size_t row, const auto value) { return out[row] = value; }, default_mask);
}
}
else
@ -783,7 +771,10 @@ size_t IPAddressDictionary::getItemsByTwoKeyColumnsShortCircuitImpl(
keys_found++;
}
else
{
set_value(i, AttributeType{});
default_mask[i] = 1;
}
}
return keys_found;
}
@ -822,7 +813,10 @@ size_t IPAddressDictionary::getItemsByTwoKeyColumnsShortCircuitImpl(
keys_found++;
}
else
{
set_value(i, AttributeType{});
default_mask[i] = 1;
}
}
return keys_found;
}
@ -893,11 +887,8 @@ void IPAddressDictionary::getItemsImpl(
}
template <typename AttributeType, typename ValueSetter>
size_t IPAddressDictionary::getItemsShortCircuitImpl(
const Attribute & attribute,
const Columns & key_columns,
ValueSetter && set_value,
IColumn::Filter & default_mask) const
void IPAddressDictionary::getItemsShortCircuitImpl(
const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, IColumn::Filter & default_mask) const
{
const auto & first_column = key_columns.front();
const size_t rows = first_column->size();
@ -909,7 +900,8 @@ size_t IPAddressDictionary::getItemsShortCircuitImpl(
keys_found = getItemsByTwoKeyColumnsShortCircuitImpl<AttributeType>(
attribute, key_columns, std::forward<ValueSetter>(set_value), default_mask);
query_count.fetch_add(rows, std::memory_order_relaxed);
return keys_found;
found_count.fetch_add(keys_found, std::memory_order_relaxed);
return;
}
auto & vec = std::get<ContainerType<AttributeType>>(attribute.maps);
@ -931,7 +923,10 @@ size_t IPAddressDictionary::getItemsShortCircuitImpl(
default_mask[i] = 0;
}
else
{
set_value(i, AttributeType{});
default_mask[i] = 1;
}
}
}
else if (type_id == TypeIndex::IPv6 || type_id == TypeIndex::FixedString)
@ -949,7 +944,10 @@ size_t IPAddressDictionary::getItemsShortCircuitImpl(
default_mask[i] = 0;
}
else
{
set_value(i, AttributeType{});
default_mask[i] = 1;
}
}
}
else
@ -957,7 +955,6 @@ size_t IPAddressDictionary::getItemsShortCircuitImpl(
query_count.fetch_add(rows, std::memory_order_relaxed);
found_count.fetch_add(keys_found, std::memory_order_relaxed);
return keys_found;
}
template <typename T>

9
src/Dictionaries/IPAddressDictionary.h
View File

@ -193,12 +193,9 @@ private:
ValueSetter && set_value,
DefaultValueExtractor & default_value_extractor) const;
template <typename AttributeType,typename ValueSetter>
size_t getItemsShortCircuitImpl(
const Attribute & attribute,
const Columns & key_columns,
ValueSetter && set_value,
IColumn::Filter & default_mask) const;
template <typename AttributeType, typename ValueSetter>
void getItemsShortCircuitImpl(
const Attribute & attribute, const Columns & key_columns, ValueSetter && set_value, IColumn::Filter & default_mask) const;
template <typename T>
void setAttributeValueImpl(Attribute & attribute, const T value); /// NOLINT

4
src/Dictionaries/PolygonDictionary.cpp
View File

@ -475,7 +475,11 @@ void IPolygonDictionary::getItemsShortCircuitImpl(
default_mask[requested_key_index] = 0;
}
else
{
auto value = AttributeType{};
set_value(value);
default_mask[requested_key_index] = 1;
}
}
query_count.fetch_add(requested_key_size, std::memory_order_relaxed);

37
src/Dictionaries/RangeHashedDictionary.cpp
View File

@ -56,27 +56,20 @@ ColumnPtr RangeHashedDictionary<dictionary_key_type>::getColumn(
if (is_short_circuit)
{
IColumn::Filter & default_mask = std::get<RefFilter>(default_or_filter).get();
size_t keys_found = 0;
if constexpr (std::is_same_v<ValueType, Array>)
{
auto * out = column.get();
keys_found = getItemsShortCircuitImpl<ValueType, false>(
attribute,
modified_key_columns,
[&](size_t, const Array & value, bool)
{
out->insert(value);
},
default_mask);
getItemsShortCircuitImpl<ValueType, false>(
attribute, modified_key_columns, [&](size_t, const Array & value, bool) { out->insert(value); }, default_mask);
}
else if constexpr (std::is_same_v<ValueType, StringRef>)
{
auto * out = column.get();
if (is_attribute_nullable)
keys_found = getItemsShortCircuitImpl<ValueType, true>(
getItemsShortCircuitImpl<ValueType, true>(
attribute,
modified_key_columns,
[&](size_t row, StringRef value, bool is_null)
@ -86,13 +79,10 @@ ColumnPtr RangeHashedDictionary<dictionary_key_type>::getColumn(
},
default_mask);
else
keys_found = getItemsShortCircuitImpl<ValueType, false>(
getItemsShortCircuitImpl<ValueType, false>(
attribute,
modified_key_columns,
[&](size_t, StringRef value, bool)
{
out->insertData(value.data, value.size);
},
[&](size_t, StringRef value, bool) { out->insertData(value.data, value.size); },
default_mask);
}
else
@ -100,7 +90,7 @@ ColumnPtr RangeHashedDictionary<dictionary_key_type>::getColumn(
auto & out = column->getData();
if (is_attribute_nullable)
keys_found = getItemsShortCircuitImpl<ValueType, true>(
getItemsShortCircuitImpl<ValueType, true>(
attribute,
modified_key_columns,
[&](size_t row, const auto value, bool is_null)
@ -110,20 +100,9 @@ ColumnPtr RangeHashedDictionary<dictionary_key_type>::getColumn(
},
default_mask);
else
keys_found = getItemsShortCircuitImpl<ValueType, false>(
attribute,
modified_key_columns,
[&](size_t row, const auto value, bool)
{
out[row] = value;
},
default_mask);
out.resize(keys_found);
getItemsShortCircuitImpl<ValueType, false>(
attribute, modified_key_columns, [&](size_t row, const auto value, bool) { out[row] = value; }, default_mask);
}
if (is_attribute_nullable)
vec_null_map_to->resize(keys_found);
}
else
{

2
src/Dictionaries/RangeHashedDictionary.h
View File

@ -245,7 +245,7 @@ private:
DefaultValueExtractor & default_value_extractor) const;
template <typename ValueType, bool is_nullable>
size_t getItemsShortCircuitImpl(
void getItemsShortCircuitImpl(
const Attribute & attribute,
const Columns & key_columns,
ValueSetterFunc<ValueType> && set_value,

6
src/Dictionaries/RangeHashedDictionaryGetItemsShortCircuitImpl.txx
View File

@ -1,7 +1,7 @@
#include <Dictionaries/RangeHashedDictionary.h>
#define INSTANTIATE_GET_ITEMS_SHORT_CIRCUIT_IMPL(DictionaryKeyType, IsNullable, ValueType) \
template size_t RangeHashedDictionary<DictionaryKeyType>::getItemsShortCircuitImpl<ValueType, IsNullable>( \
template void RangeHashedDictionary<DictionaryKeyType>::getItemsShortCircuitImpl<ValueType, IsNullable>( \
const Attribute & attribute, \
const Columns & key_columns, \
typename RangeHashedDictionary<DictionaryKeyType>::ValueSetterFunc<ValueType> && set_value, \
@ -18,7 +18,7 @@ namespace DB
template <DictionaryKeyType dictionary_key_type>
template <typename ValueType, bool is_nullable>
size_t RangeHashedDictionary<dictionary_key_type>::getItemsShortCircuitImpl(
void RangeHashedDictionary<dictionary_key_type>::getItemsShortCircuitImpl(
const Attribute & attribute,
const Columns & key_columns,
typename RangeHashedDictionary<dictionary_key_type>::ValueSetterFunc<ValueType> && set_value,
@ -120,6 +120,7 @@ size_t RangeHashedDictionary<dictionary_key_type>::getItemsShortCircuitImpl(
}
default_mask[key_index] = 1;
set_value(key_index, ValueType{}, true);
keys_extractor.rollbackCurrentKey();
}
@ -127,6 +128,5 @@ size_t RangeHashedDictionary<dictionary_key_type>::getItemsShortCircuitImpl(
query_count.fetch_add(keys_size, std::memory_order_relaxed);
found_count.fetch_add(keys_found, std::memory_order_relaxed);
return keys_found;
}
}

1
src/Dictionaries/RegExpTreeDictionary.cpp
View File

@ -807,6 +807,7 @@ std::unordered_map<String, ColumnPtr> RegExpTreeDictionary::match(
if (attributes_to_set.contains(name_))
continue;
columns[name_]->insertDefault();
default_mask.value().get()[key_idx] = 1;
}

24
src/Functions/FunctionHelpers.cpp
View File

@ -14,6 +14,7 @@ namespace DB
namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
extern const int LOGICAL_ERROR;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int SIZES_OF_ARRAYS_DONT_MATCH;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
@ -298,4 +299,27 @@ bool isDecimalOrNullableDecimal(const DataTypePtr & type)
return isDecimal(assert_cast<const DataTypeNullable *>(type.get())->getNestedType());
}
/// Note that, for historical reasons, most of the functions use the first argument size to determine which is the
/// size of all the columns. When short circuit optimization was introduced, `input_rows_count` was also added for
/// all functions, but many have not been adjusted
void checkFunctionArgumentSizes(const ColumnsWithTypeAndName & arguments, size_t input_rows_count)
{
for (size_t i = 0; i < arguments.size(); i++)
{
if (isColumnConst(*arguments[i].column))
continue;
size_t current_size = arguments[i].column->size();
if (current_size != input_rows_count)
throw Exception(
ErrorCodes::LOGICAL_ERROR,
"Expected the argument nº#{} ('{}' of type {}) to have {} rows, but it has {}",
i + 1,
arguments[i].name,
arguments[i].type->getName(),
input_rows_count,
current_size);
}
}
}

2
src/Functions/FunctionHelpers.h
View File

@ -197,4 +197,6 @@ struct NullPresence
NullPresence getNullPresense(const ColumnsWithTypeAndName & args);
bool isDecimalOrNullableDecimal(const DataTypePtr & type);
void checkFunctionArgumentSizes(const ColumnsWithTypeAndName & arguments, size_t input_rows_count);
}

22
src/Functions/FunctionsExternalDictionaries.h
View File

@ -47,7 +47,6 @@ namespace ErrorCodes
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int ILLEGAL_COLUMN;
extern const int TYPE_MISMATCH;
extern const int LOGICAL_ERROR;
}
@ -655,18 +654,6 @@ private:
result_column = if_func->build(if_args)->execute(if_args, result_type, rows);
}
#ifdef ABORT_ON_LOGICAL_ERROR
void validateShortCircuitResult(const ColumnPtr & column, const IColumn::Filter & filter) const
{
size_t expected_size = filter.size() - countBytesInFilter(filter);
size_t col_size = column->size();
if (col_size != expected_size)
throw Exception(
ErrorCodes::LOGICAL_ERROR,
"Invalid size of getColumnsOrDefaultShortCircuit result. Column has {} rows, but filter contains {} bytes.",
col_size, expected_size);
}
#endif
ColumnPtr executeDictionaryRequest(
std::shared_ptr<const IDictionary> & dictionary,
@ -696,11 +683,6 @@ private:
IColumn::Filter default_mask;
result_columns = dictionary->getColumns(attribute_names, attribute_tuple_type.getElements(), key_columns, key_types, default_mask);
#ifdef ABORT_ON_LOGICAL_ERROR
for (const auto & column : result_columns)
validateShortCircuitResult(column, default_mask);
#endif
auto [defaults_column, mask_column] =
getDefaultsShortCircuit(std::move(default_mask), result_type, last_argument);
@ -736,10 +718,6 @@ private:
IColumn::Filter default_mask;
result = dictionary->getColumn(attribute_names[0], attribute_type, key_columns, key_types, default_mask);
#ifdef ABORT_ON_LOGICAL_ERROR
validateShortCircuitResult(result, default_mask);
#endif
auto [defaults_column, mask_column] =
getDefaultsShortCircuit(std::move(default_mask), result_type, last_argument);

9
src/Functions/GatherUtils/Algorithms.h
View File

@ -440,9 +440,6 @@ void NO_INLINE conditional(SourceA && src_a, SourceB && src_b, Sink && sink, con
const UInt8 * cond_pos = condition.data();
const UInt8 * cond_end = cond_pos + condition.size();
bool a_is_short = src_a.getColumnSize() < condition.size();
bool b_is_short = src_b.getColumnSize() < condition.size();
while (cond_pos < cond_end)
{
if (*cond_pos)
@ -450,10 +447,8 @@ void NO_INLINE conditional(SourceA && src_a, SourceB && src_b, Sink && sink, con
else
writeSlice(src_b.getWhole(), sink);
if (!a_is_short || *cond_pos)
src_a.next();
if (!b_is_short || !*cond_pos)
src_b.next();
src_a.next();
src_b.next();
++cond_pos;
sink.next();

4
src/Functions/IFunction.cpp
View File

@ -110,7 +110,6 @@ void convertLowCardinalityColumnsToFull(ColumnsWithTypeAndName & args)
column.type = recursiveRemoveLowCardinality(column.type);
}
}
}
ColumnPtr IExecutableFunction::defaultImplementationForConstantArguments(
@ -277,6 +276,7 @@ ColumnPtr IExecutableFunction::executeWithoutSparseColumns(const ColumnsWithType
size_t new_input_rows_count = columns_without_low_cardinality.empty()
? input_rows_count
: columns_without_low_cardinality.front().column->size();
checkFunctionArgumentSizes(columns_without_low_cardinality, new_input_rows_count);
auto res = executeWithoutLowCardinalityColumns(columns_without_low_cardinality, dictionary_type, new_input_rows_count, dry_run);
bool res_is_constant = isColumnConst(*res);
@ -311,6 +311,8 @@ ColumnPtr IExecutableFunction::executeWithoutSparseColumns(const ColumnsWithType
ColumnPtr IExecutableFunction::execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, bool dry_run) const
{
checkFunctionArgumentSizes(arguments, input_rows_count);
bool use_default_implementation_for_sparse_columns = useDefaultImplementationForSparseColumns();
/// DataTypeFunction does not support obtaining default (isDefaultAt())
/// ColumnFunction does not support getting specific values.

13
src/Functions/IFunction.h
View File

@ -3,11 +3,12 @@
#include <Core/ColumnNumbers.h>
#include <Core/ColumnsWithTypeAndName.h>
#include <Core/Field.h>
#include <Core/ValuesWithType.h>
#include <Core/Names.h>
#include <Core/IResolvedFunction.h>
#include <Common/Exception.h>
#include <Core/Names.h>
#include <Core/ValuesWithType.h>
#include <DataTypes/IDataType.h>
#include <Functions/FunctionHelpers.h>
#include <Common/Exception.h>
#include "config.h"
@ -133,8 +134,12 @@ public:
~IFunctionBase() override = default;
virtual ColumnPtr execute( /// NOLINT
const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, bool dry_run = false) const
const ColumnsWithTypeAndName & arguments,
const DataTypePtr & result_type,
size_t input_rows_count,
bool dry_run = false) const
{
checkFunctionArgumentSizes(arguments, input_rows_count);
return prepare(arguments)->execute(arguments, result_type, input_rows_count, dry_run);
}

2
src/Functions/IFunctionAdaptors.h
View File

@ -18,11 +18,13 @@ protected:
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const final
{
checkFunctionArgumentSizes(arguments, input_rows_count);
return function->executeImpl(arguments, result_type, input_rows_count);
}
ColumnPtr executeDryRunImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const final
{
checkFunctionArgumentSizes(arguments, input_rows_count);
return function->executeImplDryRun(arguments, result_type, input_rows_count);
}

163
src/Functions/if.cpp
View File

@ -77,75 +77,17 @@ inline void fillVectorVector(const ArrayCond & cond, const ArrayA & a, const Arr
{
size_t size = cond.size();
bool a_is_short = a.size() < size;
bool b_is_short = b.size() < size;
if (a_is_short && b_is_short)
for (size_t i = 0; i < size; ++i)
{
size_t a_index = 0, b_index = 0;
for (size_t i = 0; i < size; ++i)
if constexpr (is_native_int_or_decimal_v<ResultType>)
res[i] = !!cond[i] * static_cast<ResultType>(a[i]) + (!cond[i]) * static_cast<ResultType>(b[i]);
else if constexpr (std::is_floating_point_v<ResultType>)
{
if constexpr (is_native_int_or_decimal_v<ResultType>)
res[i] = !!cond[i] * static_cast<ResultType>(a[a_index]) + (!cond[i]) * static_cast<ResultType>(b[b_index]);
else if constexpr (std::is_floating_point_v<ResultType>)
{
BRANCHFREE_IF_FLOAT(ResultType, cond[i], a[a_index], b[b_index], res[i])
}
else
res[i] = cond[i] ? static_cast<ResultType>(a[a_index]) : static_cast<ResultType>(b[b_index]);
a_index += !!cond[i];
b_index += !cond[i];
BRANCHFREE_IF_FLOAT(ResultType, cond[i], a[i], b[i], res[i])
}
}
else if (a_is_short)
{
size_t a_index = 0;
for (size_t i = 0; i < size; ++i)
else
{
if constexpr (is_native_int_or_decimal_v<ResultType>)
res[i] = !!cond[i] * static_cast<ResultType>(a[a_index]) + (!cond[i]) * static_cast<ResultType>(b[i]);
else if constexpr (std::is_floating_point_v<ResultType>)
{
BRANCHFREE_IF_FLOAT(ResultType, cond[i], a[a_index], b[i], res[i])
}
else
res[i] = cond[i] ? static_cast<ResultType>(a[a_index]) : static_cast<ResultType>(b[i]);
a_index += !!cond[i];
}
}
else if (b_is_short)
{
size_t b_index = 0;
for (size_t i = 0; i < size; ++i)
{
if constexpr (is_native_int_or_decimal_v<ResultType>)
res[i] = !!cond[i] * static_cast<ResultType>(a[i]) + (!cond[i]) * static_cast<ResultType>(b[b_index]);
else if constexpr (std::is_floating_point_v<ResultType>)
{
BRANCHFREE_IF_FLOAT(ResultType, cond[i], a[i], b[b_index], res[i])
}
else
res[i] = cond[i] ? static_cast<ResultType>(a[i]) : static_cast<ResultType>(b[b_index]);
b_index += !cond[i];
}
}
else
{
for (size_t i = 0; i < size; ++i)
{
if constexpr (is_native_int_or_decimal_v<ResultType>)
res[i] = !!cond[i] * static_cast<ResultType>(a[i]) + (!cond[i]) * static_cast<ResultType>(b[i]);
else if constexpr (std::is_floating_point_v<ResultType>)
{
BRANCHFREE_IF_FLOAT(ResultType, cond[i], a[i], b[i], res[i])
}
else
{
res[i] = cond[i] ? static_cast<ResultType>(a[i]) : static_cast<ResultType>(b[i]);
}
res[i] = cond[i] ? static_cast<ResultType>(a[i]) : static_cast<ResultType>(b[i]);
}
}
}
@ -154,37 +96,16 @@ template <typename ArrayCond, typename ArrayA, typename B, typename ArrayResult,
inline void fillVectorConstant(const ArrayCond & cond, const ArrayA & a, B b, ArrayResult & res)
{
size_t size = cond.size();
bool a_is_short = a.size() < size;
if (a_is_short)
for (size_t i = 0; i < size; ++i)
{
size_t a_index = 0;
for (size_t i = 0; i < size; ++i)
if constexpr (is_native_int_or_decimal_v<ResultType>)
res[i] = !!cond[i] * static_cast<ResultType>(a[i]) + (!cond[i]) * static_cast<ResultType>(b);
else if constexpr (std::is_floating_point_v<ResultType>)
{
if constexpr (is_native_int_or_decimal_v<ResultType>)
res[i] = !!cond[i] * static_cast<ResultType>(a[a_index]) + (!cond[i]) * static_cast<ResultType>(b);
else if constexpr (std::is_floating_point_v<ResultType>)
{
BRANCHFREE_IF_FLOAT(ResultType, cond[i], a[a_index], b, res[i])
}
else
res[i] = cond[i] ? static_cast<ResultType>(a[a_index]) : static_cast<ResultType>(b);
a_index += !!cond[i];
}
}
else
{
for (size_t i = 0; i < size; ++i)
{
if constexpr (is_native_int_or_decimal_v<ResultType>)
res[i] = !!cond[i] * static_cast<ResultType>(a[i]) + (!cond[i]) * static_cast<ResultType>(b);
else if constexpr (std::is_floating_point_v<ResultType>)
{
BRANCHFREE_IF_FLOAT(ResultType, cond[i], a[i], b, res[i])
}
else
res[i] = cond[i] ? static_cast<ResultType>(a[i]) : static_cast<ResultType>(b);
BRANCHFREE_IF_FLOAT(ResultType, cond[i], a[i], b, res[i])
}
else
res[i] = cond[i] ? static_cast<ResultType>(a[i]) : static_cast<ResultType>(b);
}
}
@ -192,37 +113,16 @@ template <typename ArrayCond, typename A, typename ArrayB, typename ArrayResult,
inline void fillConstantVector(const ArrayCond & cond, A a, const ArrayB & b, ArrayResult & res)
{
size_t size = cond.size();
bool b_is_short = b.size() < size;
if (b_is_short)
for (size_t i = 0; i < size; ++i)
{
size_t b_index = 0;
for (size_t i = 0; i < size; ++i)
if constexpr (is_native_int_or_decimal_v<ResultType>)
res[i] = !!cond[i] * static_cast<ResultType>(a) + (!cond[i]) * static_cast<ResultType>(b[i]);
else if constexpr (std::is_floating_point_v<ResultType>)
{
if constexpr (is_native_int_or_decimal_v<ResultType>)
res[i] = !!cond[i] * static_cast<ResultType>(a) + (!cond[i]) * static_cast<ResultType>(b[b_index]);
else if constexpr (std::is_floating_point_v<ResultType>)
{
BRANCHFREE_IF_FLOAT(ResultType, cond[i], a, b[b_index], res[i])
}
else
res[i] = cond[i] ? static_cast<ResultType>(a) : static_cast<ResultType>(b[b_index]);
b_index += !cond[i];
}
}
else
{
for (size_t i = 0; i < size; ++i)
{
if constexpr (is_native_int_or_decimal_v<ResultType>)
res[i] = !!cond[i] * static_cast<ResultType>(a) + (!cond[i]) * static_cast<ResultType>(b[i]);
else if constexpr (std::is_floating_point_v<ResultType>)
{
BRANCHFREE_IF_FLOAT(ResultType, cond[i], a, b[i], res[i])
}
else
res[i] = cond[i] ? static_cast<ResultType>(a) : static_cast<ResultType>(b[i]);
BRANCHFREE_IF_FLOAT(ResultType, cond[i], a, b[i], res[i])
}
else
res[i] = cond[i] ? static_cast<ResultType>(a) : static_cast<ResultType>(b[i]);
}
}
@ -880,9 +780,6 @@ private:
bool then_is_const = isColumnConst(*col_then);
bool else_is_const = isColumnConst(*col_else);
bool then_is_short = col_then->size() < cond_col->size();
bool else_is_short = col_else->size() < cond_col->size();
const auto & cond_array = cond_col->getData();
if (then_is_const && else_is_const)
@ -902,37 +799,34 @@ private:
{
const IColumn & then_nested_column = assert_cast<const ColumnConst &>(*col_then).getDataColumn();
size_t else_index = 0;
for (size_t i = 0; i < input_rows_count; ++i)
{
if (cond_array[i])
result_column->insertFrom(then_nested_column, 0);
else
result_column->insertFrom(*col_else, else_is_short ? else_index++ : i);
result_column->insertFrom(*col_else, i);
}
}
else if (else_is_const)
{
const IColumn & else_nested_column = assert_cast<const ColumnConst &>(*col_else).getDataColumn();
size_t then_index = 0;
for (size_t i = 0; i < input_rows_count; ++i)
{
if (cond_array[i])
result_column->insertFrom(*col_then, then_is_short ? then_index++ : i);
result_column->insertFrom(*col_then, i);
else
result_column->insertFrom(else_nested_column, 0);
}
}
else
{
size_t then_index = 0, else_index = 0;
for (size_t i = 0; i < input_rows_count; ++i)
{
if (cond_array[i])
result_column->insertFrom(*col_then, then_is_short ? then_index++ : i);
result_column->insertFrom(*col_then, i);
else
result_column->insertFrom(*col_else, else_is_short ? else_index++ : i);
result_column->insertFrom(*col_else, i);
}
}
@ -1125,9 +1019,6 @@ private:
if (then_is_null && else_is_null)
return result_type->createColumnConstWithDefaultValue(input_rows_count);
bool then_is_short = arg_then.column->size() < arg_cond.column->size();
bool else_is_short = arg_else.column->size() < arg_cond.column->size();
const ColumnUInt8 * cond_col = typeid_cast<const ColumnUInt8 *>(arg_cond.column.get());
const ColumnConst * cond_const_col = checkAndGetColumnConst<ColumnVector<UInt8>>(arg_cond.column.get());
@ -1146,8 +1037,6 @@ private:
{
arg_else_column = arg_else_column->convertToFullColumnIfConst();
auto result_column = IColumn::mutate(std::move(arg_else_column));
if (else_is_short)
result_column->expand(cond_col->getData(), true);
if (isColumnNullable(*result_column))
{
assert_cast<ColumnNullable &>(*result_column).applyNullMap(assert_cast<const ColumnUInt8 &>(*arg_cond.column));
@ -1193,8 +1082,6 @@ private:
{
arg_then_column = arg_then_column->convertToFullColumnIfConst();
auto result_column = IColumn::mutate(std::move(arg_then_column));
if (then_is_short)
result_column->expand(cond_col->getData(), false);
if (isColumnNullable(*result_column))
{

29
src/Functions/multiIf.cpp
View File

@ -148,11 +148,6 @@ public:
bool condition_always_true = false;
bool condition_is_nullable = false;
bool source_is_constant = false;
bool condition_is_short = false;
bool source_is_short = false;
size_t condition_index = 0;
size_t source_index = 0;
};
ColumnPtr executeImpl(const ColumnsWithTypeAndName & args, const DataTypePtr & result_type, size_t input_rows_count) const override
@ -214,12 +209,9 @@ public:
instruction.condition = cond_col;
instruction.condition_is_nullable = instruction.condition->isNullable();
}
instruction.condition_is_short = cond_col->size() < arguments[0].column->size();
}
const ColumnWithTypeAndName & source_col = arguments[source_idx];
instruction.source_is_short = source_col.column->size() < arguments[0].column->size();
if (source_col.type->equals(*return_type))
{
instruction.source = source_col.column;
@ -250,19 +242,8 @@ public:
return ColumnConst::create(std::move(res), instruction.source->size());
}
bool contains_short = false;
for (const auto & instruction : instructions)
{
if (instruction.condition_is_short || instruction.source_is_short)
{
contains_short = true;
break;
}
}
const WhichDataType which(removeNullable(result_type));
bool execute_multiif_columnar = allow_execute_multiif_columnar && !contains_short
&& instructions.size() <= std::numeric_limits<UInt8>::max()
bool execute_multiif_columnar = allow_execute_multiif_columnar && instructions.size() <= std::numeric_limits<UInt8>::max()
&& (which.isInt() || which.isUInt() || which.isFloat() || which.isDecimal() || which.isDateOrDate32OrDateTimeOrDateTime64()
|| which.isEnum() || which.isIPv4() || which.isIPv6());
@ -339,25 +320,23 @@ private:
{
bool insert = false;
size_t condition_index = instruction.condition_is_short ? instruction.condition_index++ : i;
if (instruction.condition_always_true)
insert = true;
else if (!instruction.condition_is_nullable)
insert = assert_cast<const ColumnUInt8 &>(*instruction.condition).getData()[condition_index];
insert = assert_cast<const ColumnUInt8 &>(*instruction.condition).getData()[i];
else
{
const ColumnNullable & condition_nullable = assert_cast<const ColumnNullable &>(*instruction.condition);
const ColumnUInt8 & condition_nested = assert_cast<const ColumnUInt8 &>(condition_nullable.getNestedColumn());
const NullMap & condition_null_map = condition_nullable.getNullMapData();
insert = !condition_null_map[condition_index] && condition_nested.getData()[condition_index];
insert = !condition_null_map[i] && condition_nested.getData()[i];
}
if (insert)
{
size_t source_index = instruction.source_is_short ? instruction.source_index++ : i;
if (!instruction.source_is_constant)
res->insertFrom(*instruction.source, source_index);
res->insertFrom(*instruction.source, i);
else
res->insertFrom(assert_cast<const ColumnConst &>(*instruction.source).getDataColumn(), 0);

2
src/Functions/vectorFunctions.cpp
View File

@ -1400,7 +1400,7 @@ public:
divide_result.type, input_rows_count);
auto minus_elem = minus->build({one, divide_result});
return minus_elem->execute({one, divide_result}, minus_elem->getResultType(), {});
return minus_elem->execute({one, divide_result}, minus_elem->getResultType(), input_rows_count);
}
};

1
src/Planner/PlannerActionsVisitor.cpp
View File

@ -60,6 +60,7 @@ String calculateActionNodeNameWithCastIfNeeded(const ConstantNode & constant_nod
if (constant_node.requiresCastCall())
{
/// Projection name for constants is <value>_<type> so for _cast(1, 'String') we will have _cast(1_Uint8, 'String'_String)
buffer << ", '" << constant_node.getResultType()->getName() << "'_String)";
}

14
tests/queries/0_stateless/02950_dictionary_short_circuit.sql
View File

@ -79,6 +79,10 @@ SELECT dictGetOrDefault('hashed_array_dictionary', 'v2', id+1, intDiv(NULL, id))
FROM dictionary_source_table;
SELECT dictGetOrDefault('hashed_array_dictionary', 'v3', id+1, intDiv(NULL, id))
FROM dictionary_source_table;
-- Fuzzer
SELECT dictGetOrDefault('hashed_array_dictionary', ('v1', 'v2'), toUInt128(0), (materialize(toNullable(NULL)), intDiv(1, id), intDiv(1, id))) FROM dictionary_source_table; -- { serverError TYPE_MISMATCH }
SELECT materialize(materialize(toLowCardinality(15))), dictGetOrDefault('hashed_array_dictionary', ('v1', 'v2'), 0, (intDiv(materialize(NULL), id), intDiv(1, id), intDiv(1, id))) FROM dictionary_source_table; -- { serverError TYPE_MISMATCH }
SELECT dictGetOrDefault('hashed_array_dictionary', ('v1', 'v2'), 0, (toNullable(NULL), intDiv(1, id), intDiv(1, id))) FROM dictionary_source_table; -- { serverError TYPE_MISMATCH }
DROP DICTIONARY hashed_array_dictionary;
@ -189,15 +193,15 @@ LIFETIME(3600);
SELECT 'IP TRIE dictionary';
SELECT dictGetOrDefault('ip_dictionary', 'cca2', toIPv4('202.79.32.10'), intDiv(0, id))
FROM ip_dictionary_source_table;
SELECT dictGetOrDefault('ip_dictionary', ('asn', 'cca2'), IPv6StringToNum('2a02:6b8:1::1'),
SELECT dictGetOrDefault('ip_dictionary', ('asn', 'cca2'), IPv6StringToNum('2a02:6b8:1::1'),
(intDiv(1, id), intDiv(1, id))) FROM ip_dictionary_source_table;
DROP DICTIONARY ip_dictionary;
DROP TABLE IF EXISTS polygon_dictionary_source_table;
CREATE TABLE polygon_dictionary_source_table
CREATE TABLE polygon_dictionary_source_table
(
key Array(Array(Array(Tuple(Float64, Float64)))),
key Array(Array(Array(Tuple(Float64, Float64)))),
name Nullable(String)
) ENGINE=TinyLog;
@ -258,7 +262,9 @@ LIFETIME(0)
LAYOUT(regexp_tree);
SELECT 'Regular Expression Tree dictionary';
SELECT dictGetOrDefault('regexp_dict', 'name', concat(toString(number), '/tclwebkit', toString(number)),
SELECT dictGetOrDefault('regexp_dict', 'name', concat(toString(number), '/tclwebkit', toString(number)),
intDiv(1,number)) FROM numbers(2);
-- Fuzzer
SELECT dictGetOrDefault('regexp_dict', 'name', concat('/tclwebkit', toString(number)), intDiv(1, number)) FROM numbers(2); -- { serverError ILLEGAL_DIVISION }
DROP DICTIONARY regexp_dict;
DROP TABLE regexp_dictionary_source_table;