From c3fe8bb026038f3547597c3683a530e53e13e063 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 16 Aug 2017 20:50:12 +0300 Subject: [PATCH 001/266] added FunctionArrayConcat [#CLICKHOUSE-2090] --- dbms/src/Functions/CMakeLists.txt | 1 + dbms/src/Functions/FunctionsArray.cpp | 111 ++++++++++++++++++++++++++ dbms/src/Functions/FunctionsArray.h | 19 +++++ 3 files changed, 131 insertions(+) diff --git a/dbms/src/Functions/CMakeLists.txt b/dbms/src/Functions/CMakeLists.txt index a0827afdd1e..f726e78883c 100644 --- a/dbms/src/Functions/CMakeLists.txt +++ b/dbms/src/Functions/CMakeLists.txt @@ -52,6 +52,7 @@ generate_function_register(Array FunctionRange FunctionArrayReduce FunctionArrayReverse + FunctionArrayConcat ) diff --git a/dbms/src/Functions/FunctionsArray.cpp b/dbms/src/Functions/FunctionsArray.cpp index 6fae2c7e525..4d17e7d20e9 100644 --- a/dbms/src/Functions/FunctionsArray.cpp +++ b/dbms/src/Functions/FunctionsArray.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include namespace DB @@ -2876,4 +2878,113 @@ void FunctionArrayReduce::executeImpl(Block & block, const ColumnNumbers & argum } } +/// Implementation of FunctionArrayConcat. + +FunctionPtr FunctionArrayConcat::create(const Context & context) +{ + return std::make_shared(); +} + +String FunctionArrayConcat::getName() const +{ + return name; +} + +DataTypePtr FunctionArrayConcat::getReturnTypeImpl(const DataTypes & arguments) const +{ + bool has_nullable = false; + DataTypePtr common_type; + + for (size_t i : ext::range(0, arguments.size())) + { + const auto & argument = arguments[i]; + auto data_type_array = checkAndGetDataType(argument.get()); + if (!data_type_array) + throw Exception("Argument " + toString(i) + " for function " + getName() + " must be an array but it has type " + + argument->getName() + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + DataTypePtr nested_type = data_type_array->getNestedType(); + if (nested_type->isNullable()) + { + has_nullable = true; + auto nullable_type = static_cast(nested_type.get()); + nested_type = nullable_type->getNestedType(); + } + + if (!common_type) + common_type = nested_type; + else if (!common_type->equals(*nested_type.get())) + throw Exception("Arguments for function " + getName() + " has different nested types: " + + common_type->getName() + " and " + nested_type->getName() + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + if (has_nullable) + common_type = std::make_shared(common_type); + + return std::make_shared(common_type); +} + +void FunctionArrayConcat::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) +{ + std::cerr << block.dumpStructure() << std::endl; + auto & result_column = block.getByPosition(result).column; + + ColumnPtr column_to_clone_for_result = block.getByPosition(arguments.front()).column; + bool is_nullable_result = false; + size_t size = 0; + + for (auto argument : arguments) + { + const auto & argument_column = block.safeGetByPosition(argument).column; + auto argument_column_array = typeid_cast(argument_column.get()); + if (checkColumn(&argument_column_array->getData())) + { + is_nullable_result = true; + column_to_clone_for_result = argument_column; + break; + } + size = argument_column->size(); + } + + std::vector nullable_columns_holder; + std::vector sources; + + for (auto argument : arguments) + { + auto & argument_column = block.getByPosition(argument).column; + auto argument_column_array = typeid_cast(argument_column.get()); + + if (is_nullable_result) + { + if (!checkColumn(&argument_column_array->getData())) + { + ColumnPtr null_map = std::make_shared(argument_column_array->getData().size(), 0); + argument_column = std::make_shared( + std::make_shared(argument_column_array->getDataPtr(), null_map), + argument_column_array->getOffsetsColumn() + ); + } + nullable_columns_holder.push_back(argument_column); + } + sources.emplace_back(static_cast(*argument_column.get())); + } + + result_column = column_to_clone_for_result->cloneEmpty(); + GenericArraySink sink(typeid_cast(*result_column.get()), size); + + while (!sink.isEnd()) + { + for (auto & source : sources) + { + auto slice = source.getWhole(); + std::cerr << slice.size << std::endl; + writeSlice(slice, sink); + source.next(); + } + sink.next(); + std::cerr << sink.elements.size() << std::endl; + } + std::cerr << block.dumpStructure() << std::endl; +} + } diff --git a/dbms/src/Functions/FunctionsArray.h b/dbms/src/Functions/FunctionsArray.h index e79242e4c9b..1af4df8ca84 100644 --- a/dbms/src/Functions/FunctionsArray.h +++ b/dbms/src/Functions/FunctionsArray.h @@ -1406,6 +1406,25 @@ private: }; +class FunctionArrayConcat : public IFunction +{ +public: + static constexpr auto name = "arrayConcat"; + static FunctionPtr create(const Context & context); + + String getName() const override; + + bool isVariadic() const override { return false; } + size_t getNumberOfArguments() const override { return 2; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override; + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) override; + + bool useDefaultImplementationForConstants() const override { return true; } +}; + + struct NameHas { static constexpr auto name = "has"; }; struct NameIndexOf { static constexpr auto name = "indexOf"; }; struct NameCountEqual { static constexpr auto name = "countEqual"; }; From c1479a55946d37b896dda8affbd6f83d5b66e4ad Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 17 Aug 2017 22:33:02 +0300 Subject: [PATCH 002/266] updated GatherUtils [#CLICKHOUSE-2090] --- dbms/src/Common/TypeList.h | 51 ++ dbms/src/Core/TypeListNumber.h | 10 + dbms/src/Functions/FunctionsArray.cpp | 106 ++-- dbms/src/Functions/FunctionsArray.h | 4 +- dbms/src/Functions/GatherUtils.h | 664 ++++++++++++++++++++++++-- 5 files changed, 736 insertions(+), 99 deletions(-) create mode 100644 dbms/src/Common/TypeList.h create mode 100644 dbms/src/Core/TypeListNumber.h diff --git a/dbms/src/Common/TypeList.h b/dbms/src/Common/TypeList.h new file mode 100644 index 00000000000..607f81720c0 --- /dev/null +++ b/dbms/src/Common/TypeList.h @@ -0,0 +1,51 @@ +#pragma once + +namespace DB +{ + +template +struct TypeList; + +template +struct TypeList +{ + using Head = Type; + using Tail = TypeList; +}; + +template<> +struct TypeList<> +{ +}; + +/// Append Type to TypeList +/// Usage: +/// using TypeListWithType = typename AppendToTypeList::Type; +template +struct AppendToTypeList +{ + using Type = typename AppendToTypeList::Type; +}; + +template +struct AppendToTypeList, Types ...> +{ + using Type = TypeList; +}; + +/// Apply TypeList as variadic template argument of Class. +/// Usage: +/// using ClassWithAppliedTypeList = typename ApplyTypeListForClass::Type; +template