From 0f2024306deeb9b67b631391afc9d2c70d622ffc Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sat, 3 Aug 2024 17:03:27 +0800 Subject: [PATCH 1/9] optimize array() with single arguments and map() with double arguments --- src/Functions/array/array.cpp | 28 +++++++++++++++------ src/Functions/map.cpp | 47 +++++++++++++++++++++++------------ 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index 03b51808799..202aa83216b 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -46,8 +46,10 @@ public: size_t num_elements = arguments.size(); if (num_elements == 0) + { /// We should return constant empty array. return result_type->createColumnConstWithDefaultValue(input_rows_count); + } const DataTypePtr & elem_type = static_cast(*result_type).getNestedType(); @@ -59,7 +61,6 @@ public: Columns columns_holder(num_elements); ColumnRawPtrs column_ptrs(num_elements); - for (size_t i = 0; i < num_elements; ++i) { const auto & arg = arguments[i]; @@ -76,22 +77,33 @@ public: } /// Create and fill the result array. - auto out = ColumnArray::create(elem_type->createColumn()); IColumn & out_data = out->getData(); IColumn::Offsets & out_offsets = out->getOffsets(); out_data.reserve(input_rows_count * num_elements); - out_offsets.resize(input_rows_count); + out_offsets.resize_exact(input_rows_count); IColumn::Offset current_offset = 0; - for (size_t i = 0; i < input_rows_count; ++i) + if (num_elements == 1) { - for (size_t j = 0; j < num_elements; ++j) - out_data.insertFrom(*column_ptrs[j], i); + for (size_t i = 0; i < input_rows_count; ++i) + { + ++current_offset; + out_offsets[i] = current_offset; + } + out_data.insertManyFrom(*column_ptrs[0], 0, input_rows_count); + } + else + { + for (size_t i = 0; i < input_rows_count; ++i) + { + for (size_t j = 0; j < num_elements; ++j) + out_data.insertFrom(*column_ptrs[j], i); - current_offset += num_elements; - out_offsets[i] = current_offset; + current_offset += num_elements; + out_offsets[i] = current_offset; + } } return out; diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 66cd10a3f0b..b869a475af0 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -1,14 +1,15 @@ -#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include +#include #include +#include #include @@ -123,24 +124,38 @@ public: MutableColumnPtr keys_data = key_type->createColumn(); MutableColumnPtr values_data = value_type->createColumn(); - MutableColumnPtr offsets = DataTypeNumber().createColumn(); + MutableColumnPtr offsets = DataTypeUInt64().createColumn(); size_t total_elements = input_rows_count * num_elements / 2; keys_data->reserve(total_elements); values_data->reserve(total_elements); - offsets->reserve(input_rows_count); + auto & offsets_data = assert_cast(*offsets).getData(); + offsets_data.resize_exact(input_rows_count); IColumn::Offset current_offset = 0; - for (size_t i = 0; i < input_rows_count; ++i) + if (num_elements == 2) { - for (size_t j = 0; j < num_elements; j += 2) + for (size_t i = 0; i < input_rows_count; ++i) { - keys_data->insertFrom(*column_ptrs[j], i); - values_data->insertFrom(*column_ptrs[j + 1], i); + ++current_offset; + offsets_data[i] = current_offset; } + keys_data->insertManyFrom(*column_ptrs[0], 0, input_rows_count); + values_data->insertManyFrom(*column_ptrs[1], 0, input_rows_count); + } + else + { + for (size_t i = 0; i < input_rows_count; ++i) + { + for (size_t j = 0; j < num_elements; j += 2) + { + keys_data->insertFrom(*column_ptrs[j], i); + values_data->insertFrom(*column_ptrs[j + 1], i); + } - current_offset += num_elements / 2; - offsets->insert(current_offset); + current_offset += num_elements / 2; + offsets_data[i] = current_offset; + } } auto nested_column = ColumnArray::create( From e5d4eaa4580f5921a472174d805c852e8b057165 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sat, 3 Aug 2024 21:08:21 +0800 Subject: [PATCH 2/9] fix bugs --- src/Functions/array/array.cpp | 2 +- src/Functions/map.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index 202aa83216b..a15fe86a232 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -92,7 +92,7 @@ public: ++current_offset; out_offsets[i] = current_offset; } - out_data.insertManyFrom(*column_ptrs[0], 0, input_rows_count); + out_data.insertRangeFrom(*column_ptrs[0], 0, input_rows_count); } else { diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index b869a475af0..ebfa9d4eab4 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -140,8 +140,8 @@ public: ++current_offset; offsets_data[i] = current_offset; } - keys_data->insertManyFrom(*column_ptrs[0], 0, input_rows_count); - values_data->insertManyFrom(*column_ptrs[1], 0, input_rows_count); + keys_data->insertRangeFrom(*column_ptrs[0], 0, input_rows_count); + values_data->insertRangeFrom(*column_ptrs[1], 0, input_rows_count); } else { From a498259ec8c08d4f37f77c8a6870f519bc295801 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sun, 4 Aug 2024 13:32:17 +0800 Subject: [PATCH 3/9] optimize array with mutiple arguments --- src/Functions/array/array.cpp | 212 +++++++++++++++++++++++++++++----- 1 file changed, 184 insertions(+), 28 deletions(-) diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index a15fe86a232..b25505e9ac1 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -1,10 +1,14 @@ -#include -#include +#include +#include +#include #include #include -#include -#include +#include +#include #include +#include +#include "Columns/ColumnFixedString.h" +#include "Columns/ColumnNullable.h" namespace DB @@ -43,7 +47,7 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { - size_t num_elements = arguments.size(); + const size_t num_elements = arguments.size(); if (num_elements == 0) { @@ -81,42 +85,194 @@ public: IColumn & out_data = out->getData(); IColumn::Offsets & out_offsets = out->getOffsets(); - out_data.reserve(input_rows_count * num_elements); + /// Fill out_offsets out_offsets.resize_exact(input_rows_count); - IColumn::Offset current_offset = 0; + for (size_t i = 0; i < input_rows_count; ++i) + { + current_offset += num_elements; + out_offsets[i] = current_offset; + } + + /// Fill out_data + out_data.reserve(input_rows_count * num_elements); if (num_elements == 1) - { - for (size_t i = 0; i < input_rows_count; ++i) - { - ++current_offset; - out_offsets[i] = current_offset; - } out_data.insertRangeFrom(*column_ptrs[0], 0, input_rows_count); - } else - { - for (size_t i = 0; i < input_rows_count; ++i) - { - for (size_t j = 0; j < num_elements; ++j) - out_data.insertFrom(*column_ptrs[j], i); - - current_offset += num_elements; - out_offsets[i] = current_offset; - } - } - + executeAny(column_ptrs, out_data, input_rows_count); return out; } - private: + bool executeAny(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + return false // NOLINT + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) + || executeNumber(columns, out_data, input_rows_count) || executeString(columns, out_data, input_rows_count) + || executeNullable(columns, out_data, input_rows_count) || executeTuple(columns, out_data, input_rows_count) + || executeFixedString(columns, out_data, input_rows_count) || executeGeneric(columns, out_data, input_rows_count); + } + + bool executeGeneric(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + for (size_t i = 0; i < input_rows_count; ++i) + for (const auto * column : columns) + out_data.insertFrom(*column, i); + return true; + } + + template + bool executeNumber(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + using Container = ColumnVectorOrDecimal::Container; + std::vector containers(columns.size(), nullptr); + for (size_t i = 0; i < columns.size(); ++i) + { + const ColumnVectorOrDecimal * concrete_column = checkAndGetColumn>(columns[i]); + if (!concrete_column) + return false; + + containers[i] = &concrete_column->getData(); + } + + ColumnVectorOrDecimal & concrete_out_data = assert_cast &>(out_data); + Container & out_container = concrete_out_data.getData(); + out_container.resize_exact(columns.size() * input_rows_count); + + for (size_t row_i = 0; row_i < input_rows_count; ++row_i) + { + const size_t base = row_i * columns.size(); + for (size_t col_i = 0; col_i < columns.size(); ++col_i) + out_container[base + col_i] = (*containers[col_i])[row_i]; + } + return true; + } + + bool executeString(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + size_t total_bytes = 0; + std::vector concrete_columns(columns.size(), nullptr); + for (size_t i = 0; i < columns.size(); ++i) + { + const ColumnString * concrete_column = checkAndGetColumn(columns[i]); + if (!concrete_column) + return false; + + total_bytes += concrete_column->getChars().size(); + concrete_columns[i] = concrete_column; + } + + ColumnString & concreate_out_data = assert_cast(out_data); + auto & out_chars = concreate_out_data.getChars(); + auto & out_offsets = concreate_out_data.getOffsets(); + out_chars.resize_exact(total_bytes); + out_offsets.resize_exact(input_rows_count * columns.size()); + + size_t curr_out_offset = 0; + for (size_t row_i = 0; row_i < input_rows_count; ++row_i) + { + const size_t base = row_i * columns.size(); + for (size_t col_i = 0; col_i < columns.size(); ++col_i) + { + StringRef ref = concrete_columns[col_i]->getDataAt(row_i); + memcpySmallAllowReadWriteOverflow15(&out_chars[curr_out_offset], ref.data, ref.size); + out_chars[curr_out_offset + ref.size] = 0; + + curr_out_offset += ref.size + 1; + out_offsets[base + col_i] = curr_out_offset; + } + } + return true; + } + + bool executeFixedString(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + size_t total_bytes = 0; + std::vector concrete_columns(columns.size(), nullptr); + for (size_t i = 0; i < columns.size(); ++i) + { + const ColumnFixedString * concrete_column = checkAndGetColumn(columns[i]); + if (!concrete_column) + return false; + + total_bytes += concrete_column->getChars().size(); + concrete_columns[i] = concrete_column; + } + + ColumnFixedString & concreate_out_data = assert_cast(out_data); + auto & out_chars = concreate_out_data.getChars(); + out_chars.resize_exact(total_bytes); + + const size_t n = concreate_out_data.getN(); + size_t curr_out_offset = 0; + for (size_t row_i = 0; row_i < input_rows_count; ++row_i) + { + for (size_t col_i = 0; col_i < columns.size(); ++col_i) + { + StringRef ref = concrete_columns[col_i]->getDataAt(row_i); + memcpySmallAllowReadWriteOverflow15(&out_chars[curr_out_offset], ref.data, n); + curr_out_offset += n; + } + } + return true; + } + + bool executeNullable(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + ColumnRawPtrs null_maps(columns.size(), nullptr); + ColumnRawPtrs nested_columns(columns.size(), nullptr); + for (size_t i = 0; i < columns.size(); ++i) + { + const ColumnNullable * concrete_column = checkAndGetColumn(columns[i]); + if (!concrete_column) + return false; + + null_maps[i] = &concrete_column->getNullMapColumn(); + nested_columns[i] = &concrete_column->getNestedColumn(); + } + + ColumnNullable & concreate_out_data = assert_cast(out_data); + auto & out_null_map = concreate_out_data.getNullMapColumn(); + auto & out_nested_column = concreate_out_data.getNestedColumn(); + executeAny(null_maps, out_null_map, input_rows_count); + executeAny(nested_columns, out_nested_column, input_rows_count); + return true; + } + + bool executeTuple(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + ColumnTuple * concreate_out_data = typeid_cast(&out_data); + if (!concreate_out_data) + return false; + + const size_t tuple_size = concreate_out_data->tupleSize(); + for (size_t i = 0; i < tuple_size; ++i) + { + ColumnRawPtrs elem_columns(columns.size(), nullptr); + for (size_t j = 0; j < columns.size(); ++j) + { + const ColumnTuple * concrete_column = assert_cast(columns[j]); + elem_columns[j] = &concrete_column->getColumn(i); + } + executeAny(elem_columns, concreate_out_data->getColumn(i), input_rows_count); + } + return true; + } + String getName() const override { return name; } - bool addField(DataTypePtr type_res, const Field & f, Array & arr) const; - bool use_variant_as_common_type = false; }; From 00a9c30363e16aa1893320c35052ad551b803394 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sun, 4 Aug 2024 16:16:51 +0800 Subject: [PATCH 4/9] optimize map function --- src/Functions/map.cpp | 44 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index ebfa9d4eab4..994a03374c9 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -32,11 +32,18 @@ class FunctionMap : public IFunction public: static constexpr auto name = "map"; - explicit FunctionMap(bool use_variant_as_common_type_) : use_variant_as_common_type(use_variant_as_common_type_) {} + explicit FunctionMap(ContextPtr context_) + : context(std::move(context_)) + , use_variant_as_common_type( + context->getSettingsRef().allow_experimental_variant_type && context->getSettingsRef().use_variant_as_common_type) + , function_array(FunctionFactory::instance().get("array", context)) + , function_map_from_arrays(FunctionFactory::instance().get("mapFromArrays", context)) + { + } static FunctionPtr create(ContextPtr context) { - return std::make_shared(context->getSettingsRef().allow_experimental_variant_type && context->getSettingsRef().use_variant_as_common_type); + return std::make_shared(std::move(context)); } String getName() const override @@ -97,7 +104,36 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { size_t num_elements = arguments.size(); + if (num_elements == 0) + return result_type->createColumnConstWithDefaultValue(input_rows_count); + ColumnsWithTypeAndName key_args; + ColumnsWithTypeAndName value_args; + for (size_t i = 0; i < num_elements; i += 2) + { + key_args.emplace_back(arguments[i]); + value_args.emplace_back(arguments[i+1]); + } + + const auto & result_type_map = static_cast(*result_type); + const DataTypePtr & key_type = result_type_map.getKeyType(); + const DataTypePtr & value_type = result_type_map.getValueType(); + const DataTypePtr & key_array_type = std::make_shared(key_type); + const DataTypePtr & value_array_type = std::make_shared(value_type); + + /// key_array = array(args[0], args[2]...) + ColumnPtr key_array = function_array->build(key_args)->execute(key_args, key_array_type, input_rows_count); + /// value_array = array(args[1], args[3]...) + ColumnPtr value_array = function_array->build(value_args)->execute(value_args, value_array_type, input_rows_count); + + /// result = mapFromArrays(key_array, value_array) + ColumnsWithTypeAndName map_args{{key_array, key_array_type, ""}, {value_array, value_array_type, ""}}; + return function_map_from_arrays->build(map_args)->execute(map_args, result_type, input_rows_count); + } + /* + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + size_t num_elements = arguments.size(); if (num_elements == 0) return result_type->createColumnConstWithDefaultValue(input_rows_count); @@ -164,9 +200,13 @@ public: return ColumnMap::create(nested_column); } + */ private: + ContextPtr context; bool use_variant_as_common_type = false; + FunctionOverloadResolverPtr function_array; + FunctionOverloadResolverPtr function_map_from_arrays; }; /// mapFromArrays(keys, values) is a function that allows you to make key-value pair from a pair of arrays From 122c9794a65cb6d0579ec41f6fb3b4848e5c989b Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sun, 4 Aug 2024 17:04:04 +0800 Subject: [PATCH 5/9] remove useless codes --- src/Functions/map.cpp | 71 ------------------------------------------- 1 file changed, 71 deletions(-) diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 994a03374c9..4b1e41af796 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -130,77 +130,6 @@ public: ColumnsWithTypeAndName map_args{{key_array, key_array_type, ""}, {value_array, value_array_type, ""}}; return function_map_from_arrays->build(map_args)->execute(map_args, result_type, input_rows_count); } - /* - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override - { - size_t num_elements = arguments.size(); - if (num_elements == 0) - return result_type->createColumnConstWithDefaultValue(input_rows_count); - - const auto & result_type_map = static_cast(*result_type); - const DataTypePtr & key_type = result_type_map.getKeyType(); - const DataTypePtr & value_type = result_type_map.getValueType(); - - Columns columns_holder(num_elements); - ColumnRawPtrs column_ptrs(num_elements); - - for (size_t i = 0; i < num_elements; ++i) - { - const auto & arg = arguments[i]; - const auto to_type = i % 2 == 0 ? key_type : value_type; - - ColumnPtr preprocessed_column = castColumn(arg, to_type); - preprocessed_column = preprocessed_column->convertToFullColumnIfConst(); - - columns_holder[i] = std::move(preprocessed_column); - column_ptrs[i] = columns_holder[i].get(); - } - - /// Create and fill the result map. - - MutableColumnPtr keys_data = key_type->createColumn(); - MutableColumnPtr values_data = value_type->createColumn(); - MutableColumnPtr offsets = DataTypeUInt64().createColumn(); - - size_t total_elements = input_rows_count * num_elements / 2; - keys_data->reserve(total_elements); - values_data->reserve(total_elements); - auto & offsets_data = assert_cast(*offsets).getData(); - offsets_data.resize_exact(input_rows_count); - - IColumn::Offset current_offset = 0; - if (num_elements == 2) - { - for (size_t i = 0; i < input_rows_count; ++i) - { - ++current_offset; - offsets_data[i] = current_offset; - } - keys_data->insertRangeFrom(*column_ptrs[0], 0, input_rows_count); - values_data->insertRangeFrom(*column_ptrs[1], 0, input_rows_count); - } - else - { - for (size_t i = 0; i < input_rows_count; ++i) - { - for (size_t j = 0; j < num_elements; j += 2) - { - keys_data->insertFrom(*column_ptrs[j], i); - values_data->insertFrom(*column_ptrs[j + 1], i); - } - - current_offset += num_elements / 2; - offsets_data[i] = current_offset; - } - } - - auto nested_column = ColumnArray::create( - ColumnTuple::create(Columns{std::move(keys_data), std::move(values_data)}), - std::move(offsets)); - - return ColumnMap::create(nested_column); - } - */ private: ContextPtr context; From 045551f5098e36822c69cc2bb5852d7d16cfe41b Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Mon, 5 Aug 2024 11:33:58 +0800 Subject: [PATCH 6/9] fix bugs --- src/Functions/map.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index f49d66c0329..d9643087b11 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -157,6 +157,7 @@ public: bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { From f0f1015162ddfb7ef2e8046fa9c952c5c7e6b89a Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 7 Aug 2024 10:47:13 +0800 Subject: [PATCH 7/9] fix style --- src/Functions/array/array.cpp | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index 21b1841f3ed..f193acafe06 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -172,9 +172,9 @@ private: concrete_columns[i] = concrete_column; } - ColumnString & concreate_out_data = assert_cast(out_data); - auto & out_chars = concreate_out_data.getChars(); - auto & out_offsets = concreate_out_data.getOffsets(); + ColumnString & concrete_out_data = assert_cast(out_data); + auto & out_chars = concrete_out_data.getChars(); + auto & out_offsets = concrete_out_data.getOffsets(); out_chars.resize_exact(total_bytes); out_offsets.resize_exact(input_rows_count * columns.size()); @@ -197,7 +197,6 @@ private: bool executeFixedString(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const { - size_t total_bytes = 0; std::vector concrete_columns(columns.size(), nullptr); for (size_t i = 0; i < columns.size(); ++i) { @@ -205,15 +204,16 @@ private: if (!concrete_column) return false; - total_bytes += concrete_column->getChars().size(); concrete_columns[i] = concrete_column; } - ColumnFixedString & concreate_out_data = assert_cast(out_data); - auto & out_chars = concreate_out_data.getChars(); + ColumnFixedString & concrete_out_data = assert_cast(out_data); + auto & out_chars = concrete_out_data.getChars(); + + const size_t n = concrete_out_data.getN(); + size_t total_bytes = concrete_out_data.getN() * columns.size() * input_rows_count; out_chars.resize_exact(total_bytes); - const size_t n = concreate_out_data.getN(); size_t curr_out_offset = 0; for (size_t row_i = 0; row_i < input_rows_count; ++row_i) { @@ -241,9 +241,9 @@ private: nested_columns[i] = &concrete_column->getNestedColumn(); } - ColumnNullable & concreate_out_data = assert_cast(out_data); - auto & out_null_map = concreate_out_data.getNullMapColumn(); - auto & out_nested_column = concreate_out_data.getNestedColumn(); + ColumnNullable & concrete_out_data = assert_cast(out_data); + auto & out_null_map = concrete_out_data.getNullMapColumn(); + auto & out_nested_column = concrete_out_data.getNestedColumn(); executeAny(null_maps, out_null_map, input_rows_count); executeAny(nested_columns, out_nested_column, input_rows_count); return true; @@ -251,11 +251,11 @@ private: bool executeTuple(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const { - ColumnTuple * concreate_out_data = typeid_cast(&out_data); - if (!concreate_out_data) + ColumnTuple * concrete_out_data = typeid_cast(&out_data); + if (!concrete_out_data) return false; - const size_t tuple_size = concreate_out_data->tupleSize(); + const size_t tuple_size = concrete_out_data->tupleSize(); for (size_t i = 0; i < tuple_size; ++i) { ColumnRawPtrs elem_columns(columns.size(), nullptr); @@ -264,7 +264,7 @@ private: const ColumnTuple * concrete_column = assert_cast(columns[j]); elem_columns[j] = &concrete_column->getColumn(i); } - executeAny(elem_columns, concreate_out_data->getColumn(i), input_rows_count); + executeAny(elem_columns, concrete_out_data->getColumn(i), input_rows_count); } return true; } From f1c43472d80bb1f04f0d0dfac87d985a4f33bfd5 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Mon, 9 Sep 2024 11:35:54 +0800 Subject: [PATCH 8/9] change as request --- src/Functions/array/array.cpp | 44 ++++++++++++++++++----------------- src/Functions/map.cpp | 4 ++-- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index f193acafe06..57d24b185e4 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -100,14 +100,14 @@ public: if (num_elements == 1) out_data.insertRangeFrom(*column_ptrs[0], 0, input_rows_count); else - executeAny(column_ptrs, out_data, input_rows_count); + execute(column_ptrs, out_data, input_rows_count); return out; } + private: - bool executeAny(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + bool execute(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const { - return false // NOLINT - || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) + return executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) || executeNumber(columns, out_data, input_rows_count) @@ -123,14 +123,7 @@ private: || executeFixedString(columns, out_data, input_rows_count) || executeGeneric(columns, out_data, input_rows_count); } - bool executeGeneric(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const - { - for (size_t i = 0; i < input_rows_count; ++i) - for (const auto * column : columns) - out_data.insertFrom(*column, i); - return true; - } - + template bool executeNumber(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const { @@ -178,18 +171,18 @@ private: out_chars.resize_exact(total_bytes); out_offsets.resize_exact(input_rows_count * columns.size()); - size_t curr_out_offset = 0; + size_t cur_out_offset = 0; for (size_t row_i = 0; row_i < input_rows_count; ++row_i) { const size_t base = row_i * columns.size(); for (size_t col_i = 0; col_i < columns.size(); ++col_i) { StringRef ref = concrete_columns[col_i]->getDataAt(row_i); - memcpySmallAllowReadWriteOverflow15(&out_chars[curr_out_offset], ref.data, ref.size); - out_chars[curr_out_offset + ref.size] = 0; + memcpySmallAllowReadWriteOverflow15(&out_chars[cur_out_offset], ref.data, ref.size); + out_chars[cur_out_offset + ref.size] = 0; - curr_out_offset += ref.size + 1; - out_offsets[base + col_i] = curr_out_offset; + cur_out_offset += ref.size + 1; + out_offsets[base + col_i] = cur_out_offset; } } return true; @@ -211,7 +204,7 @@ private: auto & out_chars = concrete_out_data.getChars(); const size_t n = concrete_out_data.getN(); - size_t total_bytes = concrete_out_data.getN() * columns.size() * input_rows_count; + size_t total_bytes = n * columns.size() * input_rows_count; out_chars.resize_exact(total_bytes); size_t curr_out_offset = 0; @@ -244,8 +237,8 @@ private: ColumnNullable & concrete_out_data = assert_cast(out_data); auto & out_null_map = concrete_out_data.getNullMapColumn(); auto & out_nested_column = concrete_out_data.getNestedColumn(); - executeAny(null_maps, out_null_map, input_rows_count); - executeAny(nested_columns, out_nested_column, input_rows_count); + execute(null_maps, out_null_map, input_rows_count); + execute(nested_columns, out_nested_column, input_rows_count); return true; } @@ -264,11 +257,20 @@ private: const ColumnTuple * concrete_column = assert_cast(columns[j]); elem_columns[j] = &concrete_column->getColumn(i); } - executeAny(elem_columns, concrete_out_data->getColumn(i), input_rows_count); + execute(elem_columns, concrete_out_data->getColumn(i), input_rows_count); } return true; } + bool executeGeneric(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const + { + for (size_t i = 0; i < input_rows_count; ++i) + for (const auto * column : columns) + out_data.insertFrom(*column, i); + return true; + } + + String getName() const override { return name; diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index d9643087b11..14672cfd568 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -38,7 +38,7 @@ public: static constexpr auto name = "map"; explicit FunctionMap(ContextPtr context_) - : context(std::move(context_)) + : context(context_) , use_variant_as_common_type( context->getSettingsRef().allow_experimental_variant_type && context->getSettingsRef().use_variant_as_common_type) , function_array(FunctionFactory::instance().get("array", context)) @@ -48,7 +48,7 @@ public: static FunctionPtr create(ContextPtr context) { - return std::make_shared(std::move(context)); + return std::make_shared(context); } String getName() const override From 774ec08fafc851f6c71a922dd0dde4a3f1f0d41d Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Mon, 9 Sep 2024 11:37:27 +0800 Subject: [PATCH 9/9] fix style --- src/Functions/array/array.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index 57d24b185e4..d2aedd57f99 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -123,7 +123,6 @@ private: || executeFixedString(columns, out_data, input_rows_count) || executeGeneric(columns, out_data, input_rows_count); } - template bool executeNumber(const ColumnRawPtrs & columns, IColumn & out_data, size_t input_rows_count) const {