diff --git a/cmake/find_readline_edit.cmake b/cmake/find_readline_edit.cmake index e8aa7368075..6c1e116cf96 100644 --- a/cmake/find_readline_edit.cmake +++ b/cmake/find_readline_edit.cmake @@ -1,3 +1,6 @@ +include (CMakePushCheckState) +cmake_push_check_state () + set (READLINE_PATHS "/usr/local/opt/readline/lib") # First try find custom lib for macos users (default lib without history support) find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS} NO_DEFAULT_PATH) @@ -50,3 +53,5 @@ if (LINE_EDITING_LIBS AND READLINE_INCLUDE_DIR) else () message (STATUS "Not using any library for line editing.") endif () + +cmake_pop_check_state () diff --git a/cmake/test_compiler.cmake b/cmake/test_compiler.cmake index bc92e7e4cd7..03f65a58065 100644 --- a/cmake/test_compiler.cmake +++ b/cmake/test_compiler.cmake @@ -1,4 +1,7 @@ include (CheckCXXSourceCompiles) +include (CMakePushCheckState) + +cmake_push_check_state () set (TEST_FLAG "-no-pie") set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}") @@ -14,3 +17,5 @@ set (CMAKE_REQUIRED_FLAGS "") if (HAVE_NO_PIE) set (FLAG_NO_PIE ${TEST_FLAG}) endif () + +cmake_pop_check_state () diff --git a/cmake/test_cpu.cmake b/cmake/test_cpu.cmake index 9646f9956d2..b4cb60a69ee 100644 --- a/cmake/test_cpu.cmake +++ b/cmake/test_cpu.cmake @@ -1,6 +1,9 @@ # https://software.intel.com/sites/landingpage/IntrinsicsGuide/ include (CheckCXXSourceCompiles) +include (CMakePushCheckState) + +cmake_push_check_state () # gcc -dM -E -mno-sse2 - < /dev/null | sort > gcc-dump-nosse2 # gcc -dM -E -msse2 - < /dev/null | sort > gcc-dump-sse2 @@ -58,6 +61,6 @@ if (HAVE_POPCNT AND NOT AARCH64) set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}") endif () -set (CMAKE_REQUIRED_FLAGS "") +cmake_pop_check_state () # TODO: add here sse3 test if you want use it diff --git a/contrib/libpoco/Net/cmake/test_anl.cmake b/contrib/libpoco/Net/cmake/test_anl.cmake index e1f9945514f..2beb81873a8 100644 --- a/contrib/libpoco/Net/cmake/test_anl.cmake +++ b/contrib/libpoco/Net/cmake/test_anl.cmake @@ -3,8 +3,12 @@ # include (CheckCXXSourceRuns) +include (CMakePushCheckState) + find_package (Threads) +cmake_push_check_state () + if (USE_STATIC_LIBRARIES) set (ANL_LIB_NAME "libanl.a") else () @@ -32,3 +36,5 @@ check_cxx_source_runs(" if (HAVE_GETADDRINFO_A) add_definitions (-DHAVE_GETADDRINFO_A=1) endif () + +cmake_pop_check_state () diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp index 500caac43d1..417924e3aff 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -41,11 +41,6 @@ AggregateFunctionPtr createAggregateFunctionNullVariadic(AggregateFunctionPtr & AggregateFunctionPtr createAggregateFunctionCountNotNull(const DataTypes & argument_types); -AggregateFunctionFactory::AggregateFunctionFactory() -{ -} - - void AggregateFunctionFactory::registerFunction(const String & name, Creator creator, CaseSensitiveness case_sensitiveness) { if (creator == nullptr) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.h b/dbms/src/AggregateFunctions/AggregateFunctionFactory.h index 02422b5a932..39fe59cdd4e 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.h @@ -25,7 +25,6 @@ private: using AggregateFunctions = std::unordered_map; public: - AggregateFunctionFactory(); AggregateFunctionPtr get(const String & name, const DataTypes & argument_types, int recursion_level = 0) const; AggregateFunctionPtr tryGet(const String & name, const DataTypes & argument_types) const; bool isAggregateFunctionName(const String & name, int recursion_level = 0) const; @@ -40,9 +39,6 @@ public: /// Register an aggregate function by its name. void registerFunction(const String & name, Creator creator, CaseSensitiveness case_sensitiveness = CaseSensitive); - AggregateFunctionFactory(const AggregateFunctionFactory &) = delete; - AggregateFunctionFactory & operator=(const AggregateFunctionFactory &) = delete; - private: AggregateFunctionPtr getImpl(const String & name, const DataTypes & argument_types, int recursion_level) const; diff --git a/dbms/src/Columns/ColumnConst.h b/dbms/src/Columns/ColumnConst.h index db8c95fde75..75bf588ab1d 100644 --- a/dbms/src/Columns/ColumnConst.h +++ b/dbms/src/Columns/ColumnConst.h @@ -96,7 +96,7 @@ public: using Type = T; using FieldType = typename NearestFieldType::Type; - std::string getName() const override { return "ColumnConst<" + TypeName::get() + ">"; } + std::string getName() const override { return "ColumnConst<" + String(TypeName::get()) + ">"; } bool isNumeric() const override { return IsNumber::value; } bool isFixed() const override { return IsNumber::value; } size_t sizeOfField() const override { return sizeof(T); } diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index e0729aaca02..a8651160d81 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -99,7 +99,7 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi template std::string ColumnVector::getName() const { - return "ColumnVector<" + TypeName::get() + ">"; + return "ColumnVector<" + String(TypeName::get()) + ">"; } template diff --git a/dbms/src/Common/UInt128.h b/dbms/src/Common/UInt128.h index 3c37f20d5ea..e0d8acb0991 100644 --- a/dbms/src/Common/UInt128.h +++ b/dbms/src/Common/UInt128.h @@ -21,30 +21,32 @@ struct UInt128 #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif + /// This naming assumes little endian. UInt64 low; UInt64 high; UInt128() = default; - explicit UInt128(const UInt64 rhs) : low(0), high(rhs) { } - explicit UInt128(const UInt64 low, const UInt64 high) : low(low), high(high) { } + explicit UInt128(const UInt64 rhs) : low(rhs), high() {} + explicit UInt128(const UInt64 low, const UInt64 high) : low(low), high(high) {} - bool inline operator== (const UInt128 rhs) const { return std::tie(low, high) == std::tie(rhs.low, rhs.high); } - bool inline operator!= (const UInt128 rhs) const { return !operator==(rhs); } + auto tuple() const { return std::tie(high, low); } - bool inline operator< (const UInt128 rhs) const { return std::tie(low, high) < std::tie(rhs.low, rhs.high); } - bool inline operator<= (const UInt128 rhs) const { return !rhs.operator<(*this); } - bool inline operator> (const UInt128 rhs) const { return rhs.operator<(*this); } - bool inline operator>= (const UInt128 rhs) const { return !operator<(rhs); } + bool inline operator== (const UInt128 rhs) const { return tuple() == rhs.tuple(); } + bool inline operator!= (const UInt128 rhs) const { return tuple() != rhs.tuple(); } + bool inline operator< (const UInt128 rhs) const { return tuple() < rhs.tuple(); } + bool inline operator<= (const UInt128 rhs) const { return tuple() <= rhs.tuple(); } + bool inline operator> (const UInt128 rhs) const { return tuple() > rhs.tuple(); } + bool inline operator>= (const UInt128 rhs) const { return tuple() >= rhs.tuple(); } /** Types who are stored at the moment in the database have no more than 64bits and can be handle * inside an unique UInt64. */ - template bool inline operator== (const T rhs) const { return *this == UInt128(0, rhs); } - template bool inline operator!= (const T rhs) const { return *this != UInt128(0, rhs); } - template bool inline operator>= (const T rhs) const { return *this >= UInt128(0, rhs); } - template bool inline operator> (const T rhs) const { return *this > UInt128(0, rhs); } - template bool inline operator<= (const T rhs) const { return *this <= UInt128(0, rhs); } - template bool inline operator< (const T rhs) const { return *this < UInt128(0, rhs); } + template bool inline operator== (const T rhs) const { return *this == UInt128(rhs); } + template bool inline operator!= (const T rhs) const { return *this != UInt128(rhs); } + template bool inline operator>= (const T rhs) const { return *this >= UInt128(rhs); } + template bool inline operator> (const T rhs) const { return *this > UInt128(rhs); } + template bool inline operator<= (const T rhs) const { return *this <= UInt128(rhs); } + template bool inline operator< (const T rhs) const { return *this < UInt128(rhs); } template explicit operator T() const { return static_cast(high); } @@ -55,15 +57,15 @@ struct UInt128 UInt128 & operator= (const UInt64 rhs) { low = 0; high = rhs; return *this; } }; -template bool inline operator== (T a, const UInt128 b) { return b == a; } -template bool inline operator!= (T a, const UInt128 b) { return b != a; } -template bool inline operator>= (T a, const UInt128 b) { return b.low == 0 && a >= static_cast(b.high); } -template bool inline operator> (T a, const UInt128 b) { return b.low == 0 && a > static_cast(b.high); } -template bool inline operator<= (T a, const UInt128 b) { return b.low != 0 || a <= static_cast(b.high); } -template bool inline operator< (T a, const UInt128 b) { return b.low != 0 || a < static_cast(b.high); } +template bool inline operator== (T a, const UInt128 b) { return UInt128(a) == b; } +template bool inline operator!= (T a, const UInt128 b) { return UInt128(a) != b; } +template bool inline operator>= (T a, const UInt128 b) { return UInt128(a) >= b; } +template bool inline operator> (T a, const UInt128 b) { return UInt128(a) > b; } +template bool inline operator<= (T a, const UInt128 b) { return UInt128(a) <= b; } +template bool inline operator< (T a, const UInt128 b) { return UInt128(a) < b; } template <> struct IsNumber { static constexpr bool value = true; }; -template <> struct TypeName { static std::string get() { return "UInt128"; } }; +template <> struct TypeName { static const char * get() { return "UInt128"; } }; struct UInt128Hash { diff --git a/dbms/src/Common/VirtualColumnUtils.cpp b/dbms/src/Common/VirtualColumnUtils.cpp index 49f56083985..adcde94bc71 100644 --- a/dbms/src/Common/VirtualColumnUtils.cpp +++ b/dbms/src/Common/VirtualColumnUtils.cpp @@ -158,7 +158,7 @@ bool filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & c if (!expression_ast) return false; - /// Let's parse and calculate the expression. + /// Let's analyze and calculate the expression. ExpressionAnalyzer analyzer(expression_ast, context, {}, block.getColumnsList()); ExpressionActionsPtr actions = analyzer.getActions(false); actions->execute(block); diff --git a/dbms/src/Core/ErrorCodes.cpp b/dbms/src/Core/ErrorCodes.cpp index 3c0645e69f8..f713945cb67 100644 --- a/dbms/src/Core/ErrorCodes.cpp +++ b/dbms/src/Core/ErrorCodes.cpp @@ -379,6 +379,8 @@ namespace ErrorCodes extern const int INVALID_SESSION_TIMEOUT = 374; extern const int CANNOT_DLOPEN = 375; extern const int CANNOT_PARSE_UUID = 376; + extern const int ILLEGAL_SYNTAX_FOR_DATA_TYPE = 377; + extern const int DATA_TYPE_CANNOT_HAVE_ARGUMENTS = 378; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Core/FieldVisitors.h b/dbms/src/Core/FieldVisitors.h index c6799783196..996ebdf4326 100644 --- a/dbms/src/Core/FieldVisitors.h +++ b/dbms/src/Core/FieldVisitors.h @@ -134,22 +134,22 @@ class FieldVisitorConvertToNumber : public StaticVisitor public: T operator() (const Null & x) const { - throw Exception("Cannot convert NULL to " + TypeName::get(), ErrorCodes::CANNOT_CONVERT_TYPE); + throw Exception("Cannot convert NULL to " + String(TypeName::get()), ErrorCodes::CANNOT_CONVERT_TYPE); } T operator() (const String & x) const { - throw Exception("Cannot convert String to " + TypeName::get(), ErrorCodes::CANNOT_CONVERT_TYPE); + throw Exception("Cannot convert String to " + String(TypeName::get()), ErrorCodes::CANNOT_CONVERT_TYPE); } T operator() (const Array & x) const { - throw Exception("Cannot convert Array to " + TypeName::get(), ErrorCodes::CANNOT_CONVERT_TYPE); + throw Exception("Cannot convert Array to " + String(TypeName::get()), ErrorCodes::CANNOT_CONVERT_TYPE); } T operator() (const Tuple & x) const { - throw Exception("Cannot convert Tuple to " + TypeName::get(), ErrorCodes::CANNOT_CONVERT_TYPE); + throw Exception("Cannot convert Tuple to " + String(TypeName::get()), ErrorCodes::CANNOT_CONVERT_TYPE); } T operator() (const UInt64 & x) const { return x; } diff --git a/dbms/src/Core/Types.h b/dbms/src/Core/Types.h index 99491fe0518..63621b701ff 100644 --- a/dbms/src/Core/Types.h +++ b/dbms/src/Core/Types.h @@ -41,40 +41,37 @@ template struct RemoveNullable> { using Type = T; }; template struct IsNullable { static constexpr bool value = false; }; template struct IsNullable> { static constexpr bool value = true; }; -template struct IsNumber { static constexpr bool value = false; }; +template struct IsNumber { static constexpr bool value = false; }; template struct IsNumber > { static constexpr bool value = IsNumber::value; }; -template <> struct IsNumber { static constexpr bool value = true; }; -template <> struct IsNumber { static constexpr bool value = true; }; -template <> struct IsNumber { static constexpr bool value = true; }; -template <> struct IsNumber { static constexpr bool value = true; }; -template <> struct IsNumber { static constexpr bool value = true; }; -template <> struct IsNumber { static constexpr bool value = true; }; -template <> struct IsNumber { static constexpr bool value = true; }; -template <> struct IsNumber { static constexpr bool value = true; }; -template <> struct IsNumber { static constexpr bool value = true; }; -template <> struct IsNumber { static constexpr bool value = true; }; +template <> struct IsNumber { static constexpr bool value = true; }; +template <> struct IsNumber { static constexpr bool value = true; }; +template <> struct IsNumber { static constexpr bool value = true; }; +template <> struct IsNumber { static constexpr bool value = true; }; +template <> struct IsNumber { static constexpr bool value = true; }; +template <> struct IsNumber { static constexpr bool value = true; }; +template <> struct IsNumber { static constexpr bool value = true; }; +template <> struct IsNumber { static constexpr bool value = true; }; +template <> struct IsNumber { static constexpr bool value = true; }; +template <> struct IsNumber { static constexpr bool value = true; }; template struct TypeName; -template struct TypeName> { static std::string get() { return "Nullable(" + TypeName::get() + ")"; } }; +template struct TypeName> { static const char * get() { return "Nullable"; } }; -template <> struct TypeName { static std::string get() { return "Null"; } }; +template <> struct TypeName { static const char * get() { return "Null"; } }; template <> struct TypeName> : TypeName {}; -template <> struct TypeName { static std::string get() { return "UInt8"; } }; -template <> struct TypeName { static std::string get() { return "UInt16"; } }; -template <> struct TypeName { static std::string get() { return "UInt32"; } }; -template <> struct TypeName { static std::string get() { return "UInt64"; } }; -template <> struct TypeName { static std::string get() { return "Int8"; } }; -template <> struct TypeName { static std::string get() { return "Int16"; } }; -template <> struct TypeName { static std::string get() { return "Int32"; } }; -template <> struct TypeName { static std::string get() { return "Int64"; } }; -template <> struct TypeName { static std::string get() { return "Float32"; } }; -template <> struct TypeName { static std::string get() { return "Float64"; } }; -template <> struct TypeName { static std::string get() { return "String"; } }; - -/// This type is not supported by the DBMS, but is used in some internal transformations. -template <> struct TypeName{ static std::string get() { return "long double"; } }; +template <> struct TypeName { static const char * get() { return "UInt8"; } }; +template <> struct TypeName { static const char * get() { return "UInt16"; } }; +template <> struct TypeName { static const char * get() { return "UInt32"; } }; +template <> struct TypeName { static const char * get() { return "UInt64"; } }; +template <> struct TypeName { static const char * get() { return "Int8"; } }; +template <> struct TypeName { static const char * get() { return "Int16"; } }; +template <> struct TypeName { static const char * get() { return "Int32"; } }; +template <> struct TypeName { static const char * get() { return "Int64"; } }; +template <> struct TypeName { static const char * get() { return "Float32"; } }; +template <> struct TypeName { static const char * get() { return "Float64"; } }; +template <> struct TypeName { static const char * get() { return "String"; } }; } diff --git a/dbms/src/DataStreams/ColumnGathererStream.cpp b/dbms/src/DataStreams/ColumnGathererStream.cpp index ed389c34e5a..fef47a9d175 100644 --- a/dbms/src/DataStreams/ColumnGathererStream.cpp +++ b/dbms/src/DataStreams/ColumnGathererStream.cpp @@ -85,7 +85,7 @@ Block ColumnGathererStream::readImpl() if (sources.empty()) init(); - if (row_sources_buf.eof() && !source_to_fully_copy) + if (!source_to_fully_copy && row_sources_buf.eof()) return Block(); output_block = Block{column.cloneEmpty()}; diff --git a/dbms/src/DataStreams/FilterColumnsBlockInputStream.h b/dbms/src/DataStreams/FilterColumnsBlockInputStream.h index 379ec8edc3f..dabf9ffb7bb 100644 --- a/dbms/src/DataStreams/FilterColumnsBlockInputStream.h +++ b/dbms/src/DataStreams/FilterColumnsBlockInputStream.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include namespace DB diff --git a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp index 678f30bdf9b..097d66c0349 100644 --- a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp +++ b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include #include diff --git a/dbms/src/DataStreams/MaterializingBlockInputStream.cpp b/dbms/src/DataStreams/MaterializingBlockInputStream.cpp index d753f04f6e1..fefdfb3be26 100644 --- a/dbms/src/DataStreams/MaterializingBlockInputStream.cpp +++ b/dbms/src/DataStreams/MaterializingBlockInputStream.cpp @@ -1,5 +1,4 @@ #include -#include #include #include diff --git a/dbms/src/DataStreams/MaterializingBlockOutputStream.cpp b/dbms/src/DataStreams/MaterializingBlockOutputStream.cpp index f58f52d02b1..fa5c53c8279 100644 --- a/dbms/src/DataStreams/MaterializingBlockOutputStream.cpp +++ b/dbms/src/DataStreams/MaterializingBlockOutputStream.cpp @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/dbms/src/DataStreams/NativeBlockOutputStream.cpp b/dbms/src/DataStreams/NativeBlockOutputStream.cpp index faa04803b7c..3ac70d50ec9 100644 --- a/dbms/src/DataStreams/NativeBlockOutputStream.cpp +++ b/dbms/src/DataStreams/NativeBlockOutputStream.cpp @@ -5,7 +5,6 @@ #include #include -#include #include #include #include diff --git a/dbms/src/DataStreams/PrettyBlockOutputStream.cpp b/dbms/src/DataStreams/PrettyBlockOutputStream.cpp index 87703d83116..20376557b6d 100644 --- a/dbms/src/DataStreams/PrettyBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PrettyBlockOutputStream.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.cpp b/dbms/src/DataStreams/RemoteBlockInputStream.cpp index a3ad418af6d..4709ec37759 100644 --- a/dbms/src/DataStreams/RemoteBlockInputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockInputStream.cpp @@ -102,7 +102,7 @@ void RemoteBlockInputStream::sendExternalTables() { StoragePtr cur = table.second; QueryProcessingStage::Enum stage = QueryProcessingStage::Complete; - DB::BlockInputStreams input = cur->read(cur->getColumnNamesList(), ASTPtr(), context, + BlockInputStreams input = cur->read(cur->getColumnNamesList(), {}, context, stage, DEFAULT_BLOCK_SIZE, 1); if (input.size() == 0) res.push_back(std::make_pair(std::make_shared(cur->getSampleBlock()), table.first)); diff --git a/dbms/src/DataStreams/RemoveColumnsBlockInputStream.h b/dbms/src/DataStreams/RemoveColumnsBlockInputStream.h index f84d4ada55a..c8888440e28 100644 --- a/dbms/src/DataStreams/RemoveColumnsBlockInputStream.h +++ b/dbms/src/DataStreams/RemoveColumnsBlockInputStream.h @@ -1,7 +1,6 @@ #pragma once #include -#include diff --git a/dbms/src/DataStreams/ValuesRowInputStream.cpp b/dbms/src/DataStreams/ValuesRowInputStream.cpp index c526423f47a..5c795dd123e 100644 --- a/dbms/src/DataStreams/ValuesRowInputStream.cpp +++ b/dbms/src/DataStreams/ValuesRowInputStream.cpp @@ -117,25 +117,7 @@ bool ValuesRowInputStream::read(Block & block) if (value.isNull()) { /// Check that we are indeed allowed to insert a NULL. - bool is_null_allowed = false; - - if (type.isNullable()) - is_null_allowed = true; - else - { - /// NOTE: For now we support only one level of null values, i.e. - /// there are not yet such things as Array(Nullable(Array(Nullable(T))). - /// Therefore the code below is valid within the current limitations. - const auto array_type = typeid_cast(&type); - if (array_type != nullptr) - { - const auto & nested_type = array_type->getMostNestedType(); - if (nested_type->isNullable()) - is_null_allowed = true; - } - } - - if (!is_null_allowed) + if (!type.isNullable()) throw Exception{"Expression returns value " + applyVisitor(FieldVisitorToString(), value) + ", that is out of range of type " + type.getName() + ", at: " + String(prev_istr_position, std::min(SHOW_CHARS_ON_SYNTAX_ERROR, istr.buffer().end() - prev_istr_position)), diff --git a/dbms/src/DataStreams/VerticalRowOutputStream.cpp b/dbms/src/DataStreams/VerticalRowOutputStream.cpp index ba4a41dcb78..2fae4b4dcc2 100644 --- a/dbms/src/DataStreams/VerticalRowOutputStream.cpp +++ b/dbms/src/DataStreams/VerticalRowOutputStream.cpp @@ -1,4 +1,5 @@ -#include +#include + #include #include #include diff --git a/dbms/src/DataStreams/tests/expression_stream.cpp b/dbms/src/DataStreams/tests/expression_stream.cpp index 56a23a3fcfb..d70a67c66c1 100644 --- a/dbms/src/DataStreams/tests/expression_stream.cpp +++ b/dbms/src/DataStreams/tests/expression_stream.cpp @@ -50,7 +50,7 @@ try QueryProcessingStage::Enum stage; BlockInputStreamPtr in; - in = table->read(column_names, 0, context, stage, 8192, 1)[0]; + in = table->read(column_names, {}, context, stage, 8192, 1)[0]; in = std::make_shared(in, expression); in = std::make_shared(in, 10, std::max(static_cast(0), static_cast(n) - 10)); diff --git a/dbms/src/DataStreams/tests/filter_stream.cpp b/dbms/src/DataStreams/tests/filter_stream.cpp index 2305d8e4e46..b244f37ff72 100644 --- a/dbms/src/DataStreams/tests/filter_stream.cpp +++ b/dbms/src/DataStreams/tests/filter_stream.cpp @@ -55,7 +55,7 @@ try QueryProcessingStage::Enum stage; - BlockInputStreamPtr in = table->read(column_names, 0, context, stage, 8192, 1)[0]; + BlockInputStreamPtr in = table->read(column_names, {}, context, stage, 8192, 1)[0]; in = std::make_shared(in, expression, 1); in = std::make_shared(in, 10, std::max(static_cast(0), static_cast(n) - 10)); diff --git a/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp b/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp index e6ce0045baa..aae3ae5cbb2 100644 --- a/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp +++ b/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp @@ -128,7 +128,7 @@ int main(int argc, char ** argv) QueryProcessingStage::Enum stage; - BlockInputStreamPtr in = table->read(column_names, 0, context, stage, 8192, 1)[0]; + BlockInputStreamPtr in = table->read(column_names, {}, context, stage, 8192, 1)[0]; in = std::make_shared(in, expression, 4); //in = std::make_shared(in, 10, 0); diff --git a/dbms/src/DataStreams/tests/fork_streams.cpp b/dbms/src/DataStreams/tests/fork_streams.cpp index ff74e4e5cd3..0deaaf33a6e 100644 --- a/dbms/src/DataStreams/tests/fork_streams.cpp +++ b/dbms/src/DataStreams/tests/fork_streams.cpp @@ -72,7 +72,7 @@ try QueryProcessingStage::Enum stage; - BlockInputStreamPtr in = table->read(column_names, 0, context, stage, 8192, 1)[0]; + BlockInputStreamPtr in = table->read(column_names, {}, context, stage, 8192, 1)[0]; ForkBlockInputStreams fork(in); diff --git a/dbms/src/DataStreams/tests/native_streams.cpp b/dbms/src/DataStreams/tests/native_streams.cpp index 6b62d51786e..2507ee66554 100644 --- a/dbms/src/DataStreams/tests/native_streams.cpp +++ b/dbms/src/DataStreams/tests/native_streams.cpp @@ -103,7 +103,7 @@ try if (argc == 2 && 0 == strcmp(argv[1], "read")) { QueryProcessingStage::Enum stage; - BlockInputStreamPtr in = table->read(column_names, 0, Context::createGlobal(), stage, 8192, 1)[0]; + BlockInputStreamPtr in = table->read(column_names, {}, Context::createGlobal(), stage, 8192, 1)[0]; WriteBufferFromFileDescriptor out1(STDOUT_FILENO); CompressedWriteBuffer out2(out1); NativeBlockOutputStream out3(out2, ClickHouseRevision::get()); diff --git a/dbms/src/DataStreams/tests/sorting_stream.cpp b/dbms/src/DataStreams/tests/sorting_stream.cpp index c51ef739bcd..32ca351cb42 100644 --- a/dbms/src/DataStreams/tests/sorting_stream.cpp +++ b/dbms/src/DataStreams/tests/sorting_stream.cpp @@ -145,7 +145,7 @@ try QueryProcessingStage::Enum stage; - BlockInputStreamPtr in = table->read(column_names, 0, Context::createGlobal(), stage, argc == 2 ? atoi(argv[1]) : 65536, 1)[0]; + BlockInputStreamPtr in = table->read(column_names, {}, Context::createGlobal(), stage, argc == 2 ? atoi(argv[1]) : 65536, 1)[0]; in = std::make_shared(in, sort_columns); in = std::make_shared(in, sort_columns, DEFAULT_BLOCK_SIZE, 0, 0, ""); //in = std::make_shared(in, 10, 0); diff --git a/dbms/src/DataStreams/tests/union_stream.cpp b/dbms/src/DataStreams/tests/union_stream.cpp index 069d40c7a58..32de4e6e1dd 100644 --- a/dbms/src/DataStreams/tests/union_stream.cpp +++ b/dbms/src/DataStreams/tests/union_stream.cpp @@ -35,9 +35,9 @@ void test1() QueryProcessingStage::Enum stage3; BlockInputStreams streams; - streams.emplace_back(std::make_shared(table->read(column_names, 0, context, stage1, 1, 1)[0], 30, 30000)); - streams.emplace_back(std::make_shared(table->read(column_names, 0, context, stage2, 1, 1)[0], 30, 2000)); - streams.emplace_back(std::make_shared(table->read(column_names, 0, context, stage3, 1, 1)[0], 30, 100)); + streams.emplace_back(std::make_shared(table->read(column_names, {}, context, stage1, 1, 1)[0], 30, 30000)); + streams.emplace_back(std::make_shared(table->read(column_names, {}, context, stage2, 1, 1)[0], 30, 2000)); + streams.emplace_back(std::make_shared(table->read(column_names, {}, context, stage3, 1, 1)[0], 30, 100)); UnionBlockInputStream<> union_stream(streams, nullptr, 2); @@ -84,15 +84,15 @@ void test2() BlockInputStreams streams; - BlockInputStreamPtr stream1 = std::make_shared(table->read(column_names, 0, context, stage1, 1, 1)[0], 30, 30000); + BlockInputStreamPtr stream1 = std::make_shared(table->read(column_names, {}, context, stage1, 1, 1)[0], 30, 30000); stream1 = std::make_shared(stream1, extra_info1); streams.emplace_back(stream1); - BlockInputStreamPtr stream2 = std::make_shared(table->read(column_names, 0, context, stage2, 1, 1)[0], 30, 2000); + BlockInputStreamPtr stream2 = std::make_shared(table->read(column_names, {}, context, stage2, 1, 1)[0], 30, 2000); stream2 = std::make_shared(stream2, extra_info2); streams.emplace_back(stream2); - BlockInputStreamPtr stream3 = std::make_shared(table->read(column_names, 0, context, stage3, 1, 1)[0], 30, 100); + BlockInputStreamPtr stream3 = std::make_shared(table->read(column_names, {}, context, stage3, 1, 1)[0], 30, 100); stream3 = std::make_shared(stream3, extra_info3); streams.emplace_back(stream3); diff --git a/dbms/src/DataStreams/tests/union_stream2.cpp b/dbms/src/DataStreams/tests/union_stream2.cpp index 50702e29dae..dc5785ae0d8 100644 --- a/dbms/src/DataStreams/tests/union_stream2.cpp +++ b/dbms/src/DataStreams/tests/union_stream2.cpp @@ -35,7 +35,7 @@ try StoragePtr table = context.getTable("default", "hits6"); QueryProcessingStage::Enum stage; - BlockInputStreams streams = table->read(column_names, nullptr, context, stage, settings.max_block_size, settings.max_threads); + BlockInputStreams streams = table->read(column_names, {}, context, stage, settings.max_block_size, settings.max_threads); for (size_t i = 0, size = streams.size(); i < size; ++i) streams[i] = std::make_shared(streams[i]); diff --git a/dbms/src/DataTypes/DataTypeAggregateFunction.cpp b/dbms/src/DataTypes/DataTypeAggregateFunction.cpp index 5a52b1f2eca..606a788f126 100644 --- a/dbms/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/dbms/src/DataTypes/DataTypeAggregateFunction.cpp @@ -9,11 +9,24 @@ #include #include +#include +#include +#include +#include +#include namespace DB { +namespace ErrorCodes +{ + extern const int SYNTAX_ERROR; + extern const int BAD_ARGUMENTS; + extern const int PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS; +} + + std::string DataTypeAggregateFunction::getName() const { std::stringstream stream; @@ -268,5 +281,67 @@ Field DataTypeAggregateFunction::getDefault() const } +static DataTypePtr create(const ASTPtr & arguments) +{ + String function_name; + AggregateFunctionPtr function; + DataTypes argument_types; + Array params_row; + + if (arguments->children.empty()) + throw Exception("Data type AggregateFunction requires parameters: " + "name of aggregate function and list of data types for arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (const ASTFunction * parametric = typeid_cast(arguments->children[0].get())) + { + if (parametric->parameters) + throw Exception("Unexpected level of parameters to aggregate function", ErrorCodes::SYNTAX_ERROR); + function_name = parametric->name; + + const ASTs & parameters = typeid_cast(*parametric->arguments).children; + params_row.resize(parameters.size()); + + for (size_t i = 0; i < parameters.size(); ++i) + { + const ASTLiteral * lit = typeid_cast(parameters[i].get()); + if (!lit) + throw Exception("Parameters to aggregate functions must be literals", + ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS); + + params_row[i] = lit->value; + } + } + else if (const ASTIdentifier * identifier = typeid_cast(arguments->children[0].get())) + { + function_name = identifier->name; + } + else if (typeid_cast(arguments->children[0].get())) + { + throw Exception("Aggregate function name for data type AggregateFunction must be passed as identifier (without quotes) or function", + ErrorCodes::BAD_ARGUMENTS); + } + else + throw Exception("Unexpected AST element passed as aggregate function name for data type AggregateFunction. Must be identifier or function.", + ErrorCodes::BAD_ARGUMENTS); + + for (size_t i = 1; i < arguments->children.size(); ++i) + argument_types.push_back(DataTypeFactory::instance().get(arguments->children[i])); + + if (function_name.empty()) + throw Exception("Logical error: empty name of aggregate function passed", ErrorCodes::LOGICAL_ERROR); + + function = AggregateFunctionFactory::instance().get(function_name, argument_types); + if (!params_row.empty()) + function->setParameters(params_row); + function->setArguments(argument_types); + return std::make_shared(function, argument_types, params_row); +} + +void registerDataTypeAggregateFunction(DataTypeFactory & factory) +{ + factory.registerDataType("AggregateFunction", create); +} + + } diff --git a/dbms/src/DataTypes/DataTypeAggregateFunction.h b/dbms/src/DataTypes/DataTypeAggregateFunction.h index c0518ba3bd9..5eda9fe408e 100644 --- a/dbms/src/DataTypes/DataTypeAggregateFunction.h +++ b/dbms/src/DataTypes/DataTypeAggregateFunction.h @@ -36,6 +36,8 @@ public: std::string getName() const override; + const char * getFamilyName() const override { return "AggregateFunction"; } + DataTypePtr getReturnType() const { return function->getReturnType(); }; DataTypes getArgumentsDataTypes() const { return argument_types; } diff --git a/dbms/src/DataTypes/DataTypeArray.cpp b/dbms/src/DataTypes/DataTypeArray.cpp index 018a2eb1d0e..f27a381cb67 100644 --- a/dbms/src/DataTypes/DataTypeArray.cpp +++ b/dbms/src/DataTypes/DataTypeArray.cpp @@ -8,15 +8,22 @@ #include #include +#include +#include +#include + +#include #include + namespace DB { namespace ErrorCodes { extern const int CANNOT_READ_ARRAY_FROM_TEXT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -396,24 +403,28 @@ ColumnPtr DataTypeArray::createConstColumn(size_t size, const Field & field) con } -const DataTypePtr & DataTypeArray::getMostNestedType() const +static DataTypePtr create(const ASTPtr & arguments) { - const DataTypeArray * array = this; - const IDataType * array_nested_type = array->getNestedType().get(); + if (arguments->children.size() != 1) + throw Exception("Array data type family must have exactly one argument - type of elements", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - while (true) + DataTypePtr nested_type = DataTypeFactory::instance().get(arguments->children[0]); + + if (typeid_cast(nested_type.get())) { - const DataTypeArray * type = typeid_cast(array_nested_type); - if (type == nullptr) - break; - else - { - array = type; - array_nested_type = array->getNestedType().get(); - } + /// Special case: Array(Null) is actually Array(Nullable(UInt8)). + return std::make_shared( + std::make_shared( + std::make_shared())); } - return array->getNestedType(); + return std::make_shared(nested_type); +} + + +void registerDataTypeArray(DataTypeFactory & factory) +{ + factory.registerDataType("Array", create); } } diff --git a/dbms/src/DataTypes/DataTypeArray.h b/dbms/src/DataTypes/DataTypeArray.h index 0b8358102ad..600093b26f9 100644 --- a/dbms/src/DataTypes/DataTypeArray.h +++ b/dbms/src/DataTypes/DataTypeArray.h @@ -27,6 +27,16 @@ public: return "Array(" + nested->getName() + ")"; } + const char * getFamilyName() const override + { + return "Array"; + } + + bool canBeInsideNullable() const override + { + return false; + } + DataTypePtr clone() const override { return std::make_shared(enriched_nested); @@ -86,9 +96,6 @@ public: const DataTypePtr & getNestedType() const { return nested; } const DataTypeTraits::EnrichedDataTypePtr & getEnrichedNestedType() const { return enriched_nested; } const DataTypePtr & getOffsetsType() const { return offsets; } - - /// Returns the data type found at the most nested level. - const DataTypePtr & getMostNestedType() const; }; } diff --git a/dbms/src/DataTypes/DataTypeDate.cpp b/dbms/src/DataTypes/DataTypeDate.cpp index 694a8d935ca..97cc0eeb55d 100644 --- a/dbms/src/DataTypes/DataTypeDate.cpp +++ b/dbms/src/DataTypes/DataTypeDate.cpp @@ -3,6 +3,7 @@ #include #include +#include namespace DB @@ -76,4 +77,10 @@ void DataTypeDate::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const static_cast(column).getData().push_back(value.getDayNum()); } + +void registerDataTypeDate(DataTypeFactory & factory) +{ + factory.registerSimpleDataType("Date", [] { return DataTypePtr(std::make_shared()); }, DataTypeFactory::CaseInsensitive); +} + } diff --git a/dbms/src/DataTypes/DataTypeDate.h b/dbms/src/DataTypes/DataTypeDate.h index 647d2cdf208..1e8d3db2598 100644 --- a/dbms/src/DataTypes/DataTypeDate.h +++ b/dbms/src/DataTypes/DataTypeDate.h @@ -12,6 +12,7 @@ public: bool behavesAsNumber() const override { return false; } std::string getName() const override { return "Date"; } + const char * getFamilyName() const override { return "Date"; } DataTypePtr clone() const override { return std::make_shared(); } void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; diff --git a/dbms/src/DataTypes/DataTypeDateTime.cpp b/dbms/src/DataTypes/DataTypeDateTime.cpp index 16c76c59a7c..a74918f08b6 100644 --- a/dbms/src/DataTypes/DataTypeDateTime.cpp +++ b/dbms/src/DataTypes/DataTypeDateTime.cpp @@ -3,6 +3,7 @@ #include #include +#include namespace DB @@ -76,4 +77,10 @@ void DataTypeDateTime::deserializeTextCSV(IColumn & column, ReadBuffer & istr, c static_cast(column).getData().push_back(static_cast(value)); } +void registerDataTypeDateTime(DataTypeFactory & factory) +{ + factory.registerSimpleDataType("DateTime", [] { return DataTypePtr(std::make_shared()); }, DataTypeFactory::CaseInsensitive); +} + + } diff --git a/dbms/src/DataTypes/DataTypeDateTime.h b/dbms/src/DataTypes/DataTypeDateTime.h index 05d90c4cfb6..49c2de62b16 100644 --- a/dbms/src/DataTypes/DataTypeDateTime.h +++ b/dbms/src/DataTypes/DataTypeDateTime.h @@ -12,6 +12,7 @@ public: bool behavesAsNumber() const override { return false; } std::string getName() const override { return "DateTime"; } + const char * getFamilyName() const override { return "DateTime"; } DataTypePtr clone() const override { return std::make_shared(); } void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; diff --git a/dbms/src/DataTypes/DataTypeEnum.cpp b/dbms/src/DataTypes/DataTypeEnum.cpp index 4a4ac5c3cac..478103fdeef 100644 --- a/dbms/src/DataTypes/DataTypeEnum.cpp +++ b/dbms/src/DataTypes/DataTypeEnum.cpp @@ -1,9 +1,14 @@ #include #include +#include +#include +#include +#include #include #include + namespace DB { @@ -11,6 +16,8 @@ namespace ErrorCodes { extern const int SYNTAX_ERROR; extern const int EMPTY_DATA_PASSED; + extern const int UNEXPECTED_AST_STRUCTURE; + extern const int ARGUMENT_OUT_OF_BOUND; } @@ -19,6 +26,13 @@ template <> struct EnumName { static constexpr auto value = "Enum8"; }; template <> struct EnumName { static constexpr auto value = "Enum16"; }; +template +const char * DataTypeEnum::getFamilyName() const +{ + return EnumName::value; +} + + template std::string DataTypeEnum::generateName(const Values & values) { @@ -78,12 +92,12 @@ DataTypeEnum::DataTypeEnum(const Values & values_) : values{values_} if (values.empty()) throw Exception{ "DataTypeEnum enumeration cannot be empty", - ErrorCodes::EMPTY_DATA_PASSED - }; + ErrorCodes::EMPTY_DATA_PASSED}; fillMaps(); - std::sort(std::begin(values), std::end(values), [] (auto & left, auto & right) { + std::sort(std::begin(values), std::end(values), [] (auto & left, auto & right) + { return left.second < right.second; }); @@ -286,4 +300,59 @@ Field DataTypeEnum::castToValue(const Field & value_or_name) const template class DataTypeEnum; template class DataTypeEnum; + +template +static DataTypePtr create(const ASTPtr & arguments) +{ + if (arguments->children.empty()) + throw Exception("Enum data type cannot be empty", ErrorCodes::EMPTY_DATA_PASSED); + + typename DataTypeEnum::Values values; + values.reserve(arguments->children.size()); + + using FieldType = typename DataTypeEnum::FieldType; + + /// Children must be functions 'equals' with string literal as left argument and numeric literal as right argument. + for (const ASTPtr & child : arguments->children) + { + const ASTFunction * func = typeid_cast(child.get()); + if (!func + || func->name != "equals" + || func->parameters + || !func->arguments + || func->arguments->children.size() != 2) + throw Exception("Elements of Enum data type must be of form: 'name' = number, where name is string literal and number is an integer", + ErrorCodes::UNEXPECTED_AST_STRUCTURE); + + const ASTLiteral * name_literal = typeid_cast(func->arguments->children[0].get()); + const ASTLiteral * value_literal = typeid_cast(func->arguments->children[1].get()); + + if (!name_literal + || !value_literal + || name_literal->value.getType() != Field::Types::String + || (value_literal->value.getType() != Field::Types::UInt64 && value_literal->value.getType() != Field::Types::Int64)) + throw Exception("Elements of Enum data type must be of form: 'name' = number, where name is string literal and number is an integer", + ErrorCodes::UNEXPECTED_AST_STRUCTURE); + + const String & name = name_literal->value.get(); + const auto value = value_literal->value.get::Type>(); + + if (value > std::numeric_limits::max() || value < std::numeric_limits::min()) + throw Exception{ + "Value " + toString(value) + " for element '" + name + "' exceeds range of " + EnumName::value, + ErrorCodes::ARGUMENT_OUT_OF_BOUND}; + + values.emplace_back(name, value); + } + + return std::make_shared(values); +} + + +void registerDataTypeEnum(DataTypeFactory & factory) +{ + factory.registerDataType("Enum8", create>); + factory.registerDataType("Enum16", create>); +} + } diff --git a/dbms/src/DataTypes/DataTypeEnum.h b/dbms/src/DataTypes/DataTypeEnum.h index 14dee6170c6..f0437b3070d 100644 --- a/dbms/src/DataTypes/DataTypeEnum.h +++ b/dbms/src/DataTypes/DataTypeEnum.h @@ -52,6 +52,7 @@ public: const Values & getValues() const { return values; } std::string getName() const override { return name; } + const char * getFamilyName() const override; bool isNumeric() const override { return true; } bool behavesAsNumber() const override { return true; } @@ -61,8 +62,7 @@ public: if (it == std::end(value_to_name_map)) throw Exception{ "Unexpected value " + toString(value) + " for type " + getName(), - ErrorCodes::LOGICAL_ERROR - }; + ErrorCodes::LOGICAL_ERROR}; return it->second; } diff --git a/dbms/src/DataTypes/DataTypeExpression.h b/dbms/src/DataTypes/DataTypeExpression.h index 57fcbcd231a..d9f1bbe7598 100644 --- a/dbms/src/DataTypes/DataTypeExpression.h +++ b/dbms/src/DataTypes/DataTypeExpression.h @@ -20,6 +20,7 @@ public: : argument_types(argument_types_), return_type(return_type_) {} std::string getName() const override; + const char * getFamilyName() const override { return "Expression"; } DataTypePtr clone() const override { diff --git a/dbms/src/DataTypes/DataTypeFactory.cpp b/dbms/src/DataTypes/DataTypeFactory.cpp index d63c8a60f86..3b9dd3aceb8 100644 --- a/dbms/src/DataTypes/DataTypeFactory.cpp +++ b/dbms/src/DataTypes/DataTypeFactory.cpp @@ -1,33 +1,10 @@ -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include #include -#include - -#include +#include +#include +#include +#include +#include namespace DB @@ -35,218 +12,121 @@ namespace DB namespace ErrorCodes { - extern const int ARGUMENT_OUT_OF_BOUND; + extern const int LOGICAL_ERROR; extern const int UNKNOWN_TYPE; - extern const int NESTED_TYPE_TOO_DEEP; - extern const int PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS; - extern const int SYNTAX_ERROR; - extern const int BAD_ARGUMENTS; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_SYNTAX_FOR_DATA_TYPE; + extern const int UNEXPECTED_AST_STRUCTURE; + extern const int DATA_TYPE_CANNOT_HAVE_ARGUMENTS; } +DataTypePtr DataTypeFactory::get(const String & full_name) const +{ + ParserIdentifierWithOptionalParameters parser; + ASTPtr ast = parseQuery(parser, full_name.data(), full_name.data() + full_name.size(), "data type"); + return get(ast); +} + +DataTypePtr DataTypeFactory::get(const ASTPtr & ast) const +{ + if (const ASTFunction * func = typeid_cast(ast.get())) + { + if (func->parameters) + throw Exception("Data type cannot have multiple parenthesed parameters.", ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE); + return get(func->name, func->arguments); + } + + if (const ASTIdentifier * ident = typeid_cast(ast.get())) + { + return get(ident->name, {}); + } + + throw Exception("Unexpected AST element for data type.", ErrorCodes::UNEXPECTED_AST_STRUCTURE); +} + +DataTypePtr DataTypeFactory::get(const String & family_name, const ASTPtr & parameters) const +{ + { + DataTypesDictionary::const_iterator it = data_types.find(family_name); + if (data_types.end() != it) + return it->second(parameters); + } + + { + String family_name_lowercase = Poco::toLower(family_name); + DataTypesDictionary::const_iterator it = case_insensitive_data_types.find(family_name_lowercase); + if (case_insensitive_data_types.end() != it) + return it->second(parameters); + } + + throw Exception("Unknown data type family: " + family_name, ErrorCodes::UNKNOWN_TYPE); +} + + +void DataTypeFactory::registerDataType(const String & family_name, Creator creator, CaseSensitiveness case_sensitiveness) +{ + if (creator == nullptr) + throw Exception("DataTypeFactory: the data type family " + family_name + " has been provided " + " a null constructor", ErrorCodes::LOGICAL_ERROR); + + if (!data_types.emplace(family_name, creator).second) + throw Exception("DataTypeFactory: the data type family name '" + family_name + "' is not unique", + ErrorCodes::LOGICAL_ERROR); + + String family_name_lowercase = Poco::toLower(family_name); + + if (case_sensitiveness == CaseInsensitive + && !case_insensitive_data_types.emplace(family_name_lowercase, creator).second) + throw Exception("DataTypeFactory: the case insensitive data type family name '" + family_name + "' is not unique", + ErrorCodes::LOGICAL_ERROR); +} + + +void DataTypeFactory::registerSimpleDataType(const String & name, SimpleCreator creator, CaseSensitiveness case_sensitiveness) +{ + if (creator == nullptr) + throw Exception("DataTypeFactory: the data type " + name + " has been provided " + " a null constructor", ErrorCodes::LOGICAL_ERROR); + + registerDataType(name, [name, creator](const ASTPtr & ast) + { + if (ast) + throw Exception("Data type " + name + " cannot have arguments", ErrorCodes::DATA_TYPE_CANNOT_HAVE_ARGUMENTS); + return creator(); + }, case_sensitiveness); +} + + +void registerDataTypeNumbers(DataTypeFactory & factory); +void registerDataTypeDate(DataTypeFactory & factory); +void registerDataTypeDateTime(DataTypeFactory & factory); +void registerDataTypeString(DataTypeFactory & factory); +void registerDataTypeFixedString(DataTypeFactory & factory); +void registerDataTypeEnum(DataTypeFactory & factory); +void registerDataTypeArray(DataTypeFactory & factory); +void registerDataTypeTuple(DataTypeFactory & factory); +void registerDataTypeNullable(DataTypeFactory & factory); +void registerDataTypeNull(DataTypeFactory & factory); +void registerDataTypeUUID(DataTypeFactory & factory); +void registerDataTypeAggregateFunction(DataTypeFactory & factory); +void registerDataTypeNested(DataTypeFactory & factory); + + DataTypeFactory::DataTypeFactory() - : non_parametric_data_types - { - {"UInt8", std::make_shared()}, - {"UInt16", std::make_shared()}, - {"UInt32", std::make_shared()}, - {"UInt64", std::make_shared()}, - {"Int8", std::make_shared()}, - {"Int16", std::make_shared()}, - {"Int32", std::make_shared()}, - {"Int64", std::make_shared()}, - {"Float32", std::make_shared()}, - {"Float64", std::make_shared()}, - {"Date", std::make_shared()}, - {"DateTime", std::make_shared()}, - {"UUID", std::make_shared()}, - {"String", std::make_shared()}, - {"Null", std::make_shared()} - } { + registerDataTypeNumbers(*this); + registerDataTypeDate(*this); + registerDataTypeDateTime(*this); + registerDataTypeString(*this); + registerDataTypeFixedString(*this); + registerDataTypeEnum(*this); + registerDataTypeArray(*this); + registerDataTypeTuple(*this); + registerDataTypeNullable(*this); + registerDataTypeNull(*this); + registerDataTypeUUID(*this); + registerDataTypeAggregateFunction(*this); + registerDataTypeNested(*this); } - -template -inline DataTypePtr parseEnum(const String & name, const String & base_name, const String & parameters) -{ - ParserList parser{std::make_unique(), std::make_unique(TokenType::Comma), false}; - - ASTPtr elements = parseQuery(parser, parameters.data(), parameters.data() + parameters.size(), "parameters for enum type " + name); - - typename DataTypeEnum::Values values; - values.reserve(elements->children.size()); - - using FieldType = typename DataTypeEnum::FieldType; - - for (const auto & element : typeid_cast(*elements).children) - { - const auto & e = static_cast(*element); - const auto value = e.value.get::Type>(); - - if (value > std::numeric_limits::max() || value < std::numeric_limits::min()) - throw Exception{ - "Value " + applyVisitor(FieldVisitorToString{}, e.value) + " for element '" + e.name + "' exceeds range of " + base_name, - ErrorCodes::ARGUMENT_OUT_OF_BOUND}; - - values.emplace_back(e.name, value); - } - - return std::make_shared(values); -} - - -DataTypePtr DataTypeFactory::get(const String & name) const -{ - return getImpl(name, true); -} - - -DataTypePtr DataTypeFactory::getImpl(const String & name, bool allow_nullable) const -{ - NonParametricDataTypes::const_iterator it = non_parametric_data_types.find(name); - if (it != non_parametric_data_types.end()) - return it->second; - - Poco::RegularExpression::MatchVec matches; - if (fixed_string_regexp.match(name, 0, matches) && matches.size() == 2) - return std::make_shared(parse(name.data() + matches[1].offset, matches[1].length)); - - if (nested_regexp.match(name, 0, matches) && matches.size() == 3) - { - String base_name(name.data() + matches[1].offset, matches[1].length); - String parameters(name.data() + matches[2].offset, matches[2].length); - - if (base_name == "Nullable") - { - if (!allow_nullable) - throw Exception{"A Nullable type cannot contain another Nullable type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - - return std::make_shared(getImpl(parameters, false)); - } - - if (base_name == "Array") - { - if (parameters == "Null") - { - /// Special case: Array(Null) is actually Array(Nullable(UInt8)). - return std::make_shared( - std::make_shared( - std::make_shared())); - } - else - return std::make_shared(getImpl(parameters, allow_nullable)); - } - - if (base_name == "AggregateFunction") - { - String function_name; - AggregateFunctionPtr function; - DataTypes argument_types; - Array params_row; - - ParserExpressionList args_parser(false); - ASTPtr args_ast = parseQuery(args_parser, parameters.data(), parameters.data() + parameters.size(), "parameters for data type " + name); - ASTExpressionList & args_list = typeid_cast(*args_ast); - - if (args_list.children.empty()) - throw Exception("Data type AggregateFunction requires parameters: " - "name of aggregate function and list of data types for arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - if (ASTFunction * parametric = typeid_cast(args_list.children[0].get())) - { - if (parametric->parameters) - throw Exception("Unexpected level of parameters to aggregate function", ErrorCodes::SYNTAX_ERROR); - function_name = parametric->name; - - ASTs & parameters = typeid_cast(*parametric->arguments).children; - params_row.resize(parameters.size()); - - for (size_t i = 0; i < parameters.size(); ++i) - { - ASTLiteral * lit = typeid_cast(parameters[i].get()); - if (!lit) - throw Exception("Parameters to aggregate functions must be literals", - ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS); - - params_row[i] = lit->value; - } - } - else if (ASTIdentifier * identifier = typeid_cast(args_list.children[0].get())) - { - function_name = identifier->name; - } - else if (typeid_cast(args_list.children[0].get())) - { - throw Exception("Aggregate function name for data type AggregateFunction must be passed as identifier (without quotes) or function", - ErrorCodes::BAD_ARGUMENTS); - } - else - throw Exception("Unexpected AST element passed as aggregate function name for data type AggregateFunction. Must be identifier or function.", - ErrorCodes::BAD_ARGUMENTS); - - for (size_t i = 1; i < args_list.children.size(); ++i) - argument_types.push_back(getImpl( - std::string{args_list.children[i]->range.first, args_list.children[i]->range.second}, allow_nullable)); - - if (function_name.empty()) - throw Exception("Logical error: empty name of aggregate function passed", ErrorCodes::LOGICAL_ERROR); - - function = AggregateFunctionFactory::instance().get(function_name, argument_types); - if (!params_row.empty()) - function->setParameters(params_row); - function->setArguments(argument_types); - return std::make_shared(function, argument_types, params_row); - } - - if (base_name == "Nested") - { - ParserNameTypePairList columns_p; - ASTPtr columns_ast = parseQuery(columns_p, parameters.data(), parameters.data() + parameters.size(), "parameters for data type " + name); - - NamesAndTypesListPtr columns = std::make_shared(); - - ASTExpressionList & columns_list = typeid_cast(*columns_ast); - for (ASTs::iterator it = columns_list.children.begin(); it != columns_list.children.end(); ++it) - { - ASTNameTypePair & name_and_type_pair = typeid_cast(**it); - StringRange type_range = name_and_type_pair.type->range; - DataTypePtr type = getImpl(String(type_range.first, type_range.second - type_range.first), allow_nullable); - if (typeid_cast(type.get())) - throw Exception("Nested inside Nested is not allowed", ErrorCodes::NESTED_TYPE_TOO_DEEP); - columns->push_back(NameAndTypePair( - name_and_type_pair.name, - type)); - } - - return std::make_shared(columns); - } - - if (base_name == "Tuple") - { - ParserExpressionList columns_p(false); - ASTPtr columns_ast = parseQuery(columns_p, parameters.data(), parameters.data() + parameters.size(), "parameters for data type " + name); - - auto & columns_list = typeid_cast(*columns_ast); - const auto elems = ext::map(columns_list.children, [&] (const ASTPtr & elem_ast) { - return getImpl(String(elem_ast->range.first, elem_ast->range.second), allow_nullable); - }); - - return std::make_shared(elems); - } - - if (base_name == "Enum8") - return parseEnum(name, base_name, parameters); - - if (base_name == "Enum16") - return parseEnum(name, base_name, parameters); - - throw Exception("Unknown type " + base_name, ErrorCodes::UNKNOWN_TYPE); - } - - throw Exception("Unknown type " + name, ErrorCodes::UNKNOWN_TYPE); -} - - } diff --git a/dbms/src/DataTypes/DataTypeFactory.h b/dbms/src/DataTypes/DataTypeFactory.h index 7b63283b5ca..c1b7184448b 100644 --- a/dbms/src/DataTypes/DataTypeFactory.h +++ b/dbms/src/DataTypes/DataTypeFactory.h @@ -1,32 +1,57 @@ #pragma once -#include -#include -#include +#include +#include +#include #include +#include namespace DB { -/** Creates data type by its name (possibly name contains parameters in parens). +class IDataType; +using DataTypePtr = std::shared_ptr; + +class IAST; +using ASTPtr = std::shared_ptr; + + +/** Creates a data type by name of data type family and parameters. */ -class DataTypeFactory : public ext::singleton +class DataTypeFactory final : public ext::singleton { +private: + using Creator = std::function; + using SimpleCreator = std::function; + using DataTypesDictionary = std::unordered_map; + public: - DataTypeFactory(); - DataTypePtr get(const String & name) const; + DataTypePtr get(const String & full_name) const; + DataTypePtr get(const String & family_name, const ASTPtr & parameters) const; + DataTypePtr get(const ASTPtr & ast) const; + + /// For compatibility with SQL, it's possible to specify that certain data type name is case insensitive. + enum CaseSensitiveness + { + CaseSensitive, + CaseInsensitive + }; + + /// Register a type family by its name. + void registerDataType(const String & family_name, Creator creator, CaseSensitiveness case_sensitiveness = CaseSensitive); + + /// Register a simple data type, that have no parameters. + void registerSimpleDataType(const String & name, SimpleCreator creator, CaseSensitiveness case_sensitiveness = CaseSensitive); private: - DataTypePtr getImpl(const String & name, bool allow_nullable) const; + DataTypesDictionary data_types; - using NonParametricDataTypes = std::map; - NonParametricDataTypes non_parametric_data_types; + /// Case insensitive data types will be additionally added here with lowercased name. + DataTypesDictionary case_insensitive_data_types; - Poco::RegularExpression fixed_string_regexp {R"--(^FixedString\s*\(\s*(\d+)\s*\)$)--"}; - - Poco::RegularExpression nested_regexp {R"--(^(\w+)\s*\(\s*(.+)\s*\)$)--", - Poco::RegularExpression::RE_MULTILINE | Poco::RegularExpression::RE_DOTALL}; + DataTypeFactory(); + friend class ext::singleton; }; } diff --git a/dbms/src/DataTypes/DataTypeFixedString.cpp b/dbms/src/DataTypes/DataTypeFixedString.cpp index 6893d5b3ebd..20751f694c9 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.cpp +++ b/dbms/src/DataTypes/DataTypeFixedString.cpp @@ -6,13 +6,18 @@ #include #include +#include #include #include #include +#include +#include + #include + namespace DB { @@ -20,6 +25,8 @@ namespace ErrorCodes { extern const int CANNOT_READ_ALL_DATA; extern const int TOO_LARGE_STRING_SIZE; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int UNEXPECTED_AST_STRUCTURE; } @@ -204,4 +211,23 @@ ColumnPtr DataTypeFixedString::createConstColumn(size_t size, const Field & fiel return std::make_shared(size, get(field), clone()); } + +static DataTypePtr create(const ASTPtr & arguments) +{ + if (arguments->children.size() != 1) + throw Exception("FixedString data type family must have exactly one argument - size in bytes", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const ASTLiteral * argument = typeid_cast(arguments->children[0].get()); + if (!argument || argument->value.getType() != Field::Types::UInt64 || argument->value.get() == 0) + throw Exception("FixedString data type family must have a number (positive integer) as its argument", ErrorCodes::UNEXPECTED_AST_STRUCTURE); + + return std::make_shared(argument->value.get()); +} + + +void registerDataTypeFixedString(DataTypeFactory & factory) +{ + factory.registerDataType("FixedString", create); +} + } diff --git a/dbms/src/DataTypes/DataTypeFixedString.h b/dbms/src/DataTypes/DataTypeFixedString.h index 6588111853e..d72526d3c62 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.h +++ b/dbms/src/DataTypes/DataTypeFixedString.h @@ -26,6 +26,8 @@ public: std::string getName() const override; + const char * getFamilyName() const override { return "FixedString"; } + DataTypePtr clone() const override { return std::make_shared(n); diff --git a/dbms/src/DataTypes/DataTypeNested.cpp b/dbms/src/DataTypes/DataTypeNested.cpp index 494234d36da..06320329a3c 100644 --- a/dbms/src/DataTypes/DataTypeNested.cpp +++ b/dbms/src/DataTypes/DataTypeNested.cpp @@ -5,6 +5,10 @@ #include #include +#include + +#include +#include namespace DB @@ -13,6 +17,8 @@ namespace DB namespace ErrorCodes { extern const int INVALID_NESTED_NAME; + extern const int EMPTY_DATA_PASSED; + extern const int NESTED_TYPE_TOO_DEEP; } @@ -90,4 +96,30 @@ NamesAndTypesListPtr DataTypeNested::expandNestedColumns(const NamesAndTypesList return columns; } + +static DataTypePtr create(const ASTPtr & arguments) +{ + if (arguments->children.empty()) + throw Exception("Nested structure cannot be empty", ErrorCodes::EMPTY_DATA_PASSED); + + NamesAndTypesListPtr columns = std::make_shared(); + + for (const auto & child : arguments->children) + { + const ASTNameTypePair & name_and_type_pair = typeid_cast(*child); + DataTypePtr type = DataTypeFactory::instance().get(name_and_type_pair.type); + if (typeid_cast(type.get())) + throw Exception("Nested inside Nested is not allowed", ErrorCodes::NESTED_TYPE_TOO_DEEP); + columns->emplace_back(name_and_type_pair.name, type); + } + + return std::make_shared(columns); +} + + +void registerDataTypeNested(DataTypeFactory & factory) +{ + factory.registerDataType("Nested", create); +} + } diff --git a/dbms/src/DataTypes/DataTypeNested.h b/dbms/src/DataTypes/DataTypeNested.h index 2f02e644996..0af35a7912e 100644 --- a/dbms/src/DataTypes/DataTypeNested.h +++ b/dbms/src/DataTypes/DataTypeNested.h @@ -20,6 +20,7 @@ public: DataTypeNested(NamesAndTypesListPtr nested_); std::string getName() const override; + const char * getFamilyName() const override { return "Nested"; } DataTypePtr clone() const override { diff --git a/dbms/src/DataTypes/DataTypeNull.cpp b/dbms/src/DataTypes/DataTypeNull.cpp index cb1ba7272ef..b3c0e58e110 100644 --- a/dbms/src/DataTypes/DataTypeNull.cpp +++ b/dbms/src/DataTypes/DataTypeNull.cpp @@ -1,11 +1,13 @@ -#include - #include #include #include #include #include +#include + +#include +#include namespace DB @@ -118,4 +120,10 @@ void DataTypeNull::deserializeTextJSON(IColumn & column, ReadBuffer & istr) cons assertString("null", istr); } + +void registerDataTypeNull(DataTypeFactory & factory) +{ + factory.registerSimpleDataType("Null", [] { return DataTypePtr(std::make_shared()); }); +} + } diff --git a/dbms/src/DataTypes/DataTypeNull.h b/dbms/src/DataTypes/DataTypeNull.h index d58ecfff77d..c312ddc3428 100644 --- a/dbms/src/DataTypes/DataTypeNull.h +++ b/dbms/src/DataTypes/DataTypeNull.h @@ -21,6 +21,11 @@ public: return "Null"; } + const char * getFamilyName() const override + { + return "Null"; + } + bool isNull() const override { return true; @@ -31,6 +36,11 @@ public: return true; } + bool canBeInsideNullable() const override + { + return false; + } + DataTypePtr clone() const override { return std::make_shared(); diff --git a/dbms/src/DataTypes/DataTypeNullable.cpp b/dbms/src/DataTypes/DataTypeNullable.cpp index 8c475a080f6..07148e458cb 100644 --- a/dbms/src/DataTypes/DataTypeNullable.cpp +++ b/dbms/src/DataTypes/DataTypeNullable.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,11 +7,19 @@ #include #include #include +#include +#include namespace DB { +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + DataTypeNullable::DataTypeNullable(DataTypePtr nested_data_type_) : nested_data_type{nested_data_type_} @@ -236,4 +245,24 @@ ColumnPtr DataTypeNullable::createConstColumn(size_t size, const Field & field) std::make_shared(size, 0)); } + +static DataTypePtr create(const ASTPtr & arguments) +{ + if (arguments->children.size() != 1) + throw Exception("Nullable data type family must have exactly one argument - nested type", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + DataTypePtr nested_type = DataTypeFactory::instance().get(arguments->children[0]); + + if (!nested_type->canBeInsideNullable()) + throw Exception("Nested type " + nested_type->getName() + " cannot be inside Nullable type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(nested_type); +} + + +void registerDataTypeNullable(DataTypeFactory & factory) +{ + factory.registerDataType("Nullable", create); +} + } diff --git a/dbms/src/DataTypes/DataTypeNullable.h b/dbms/src/DataTypes/DataTypeNullable.h index 5a43899b652..bb9c6bc0cac 100644 --- a/dbms/src/DataTypes/DataTypeNullable.h +++ b/dbms/src/DataTypes/DataTypeNullable.h @@ -13,11 +13,13 @@ class DataTypeNullable final : public IDataType public: DataTypeNullable(DataTypePtr nested_data_type_); std::string getName() const override { return "Nullable(" + nested_data_type->getName() + ")"; } + const char * getFamilyName() const override { return "Nullable"; } bool isNullable() const override { return true; } bool isNumeric() const override { return nested_data_type->isNumeric(); } /// TODO Absolutely wrong. bool isNumericNotNullable() const override { return false; } bool behavesAsNumber() const override { return nested_data_type->behavesAsNumber(); } /// TODO Absolutely wrong. + bool canBeInsideNullable() const override { return false; } DataTypePtr clone() const override { return std::make_shared(nested_data_type->clone()); } diff --git a/dbms/src/DataTypes/DataTypeNumberBase.cpp b/dbms/src/DataTypes/DataTypeNumberBase.cpp index 9b570886177..f226dd0d83e 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.cpp +++ b/dbms/src/DataTypes/DataTypeNumberBase.cpp @@ -259,4 +259,5 @@ template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; + } diff --git a/dbms/src/DataTypes/DataTypeNumberBase.h b/dbms/src/DataTypes/DataTypeNumberBase.h index 267468a2e66..a8aa79e2a57 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.h +++ b/dbms/src/DataTypes/DataTypeNumberBase.h @@ -6,8 +6,7 @@ namespace DB { -/** Implements part of the IDataType interface, common to all numbers - * - input and output in text form. +/** Implements part of the IDataType interface, common to all numbers and for Date and DateTime. */ template class DataTypeNumberBase : public IDataType @@ -16,6 +15,7 @@ public: using FieldType = T; std::string getName() const override { return TypeName::get(); } + const char * getFamilyName() const override { return TypeName::get(); } bool isNumeric() const override { return true; } bool behavesAsNumber() const override { return true; } diff --git a/dbms/src/DataTypes/DataTypeSet.h b/dbms/src/DataTypes/DataTypeSet.h index 8e4f11b100d..41039fbe722 100644 --- a/dbms/src/DataTypes/DataTypeSet.h +++ b/dbms/src/DataTypes/DataTypeSet.h @@ -7,12 +7,13 @@ namespace DB { /** The data type corresponding to the set of values in the IN section. - * Used only as an intermediate option when evaluating expressions. + * Used only as an intermediate when evaluating expressions. */ class DataTypeSet final : public IDataTypeDummy { public: std::string getName() const override { return "Set"; } + const char * getFamilyName() const override { return "Set"; } DataTypePtr clone() const override { return std::make_shared(); } }; diff --git a/dbms/src/DataTypes/DataTypeString.cpp b/dbms/src/DataTypes/DataTypeString.cpp index 57e7cb47460..1bf4464b5af 100644 --- a/dbms/src/DataTypes/DataTypeString.cpp +++ b/dbms/src/DataTypes/DataTypeString.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -301,4 +302,25 @@ ColumnPtr DataTypeString::createConstColumn(size_t size, const Field & field) co return std::make_shared(size, get(field)); } + +void registerDataTypeString(DataTypeFactory & factory) +{ + auto creator = static_cast([] { return DataTypePtr(std::make_shared()); }); + + factory.registerSimpleDataType("String", creator); + + /// These synonims are added for compatibility. + + factory.registerSimpleDataType("CHAR", creator, DataTypeFactory::CaseInsensitive); + factory.registerSimpleDataType("VARCHAR", creator, DataTypeFactory::CaseInsensitive); + factory.registerSimpleDataType("TEXT", creator, DataTypeFactory::CaseInsensitive); + factory.registerSimpleDataType("TINYTEXT", creator, DataTypeFactory::CaseInsensitive); + factory.registerSimpleDataType("MEDIUMTEXT", creator, DataTypeFactory::CaseInsensitive); + factory.registerSimpleDataType("LONGTEXT", creator, DataTypeFactory::CaseInsensitive); + factory.registerSimpleDataType("BLOB", creator, DataTypeFactory::CaseInsensitive); + factory.registerSimpleDataType("TINYBLOB", creator, DataTypeFactory::CaseInsensitive); + factory.registerSimpleDataType("MEDIUMBLOB", creator, DataTypeFactory::CaseInsensitive); + factory.registerSimpleDataType("LONGBLOB", creator, DataTypeFactory::CaseInsensitive); +} + } diff --git a/dbms/src/DataTypes/DataTypeString.h b/dbms/src/DataTypes/DataTypeString.h index 3e17d686f7d..e8974c2ce69 100644 --- a/dbms/src/DataTypes/DataTypeString.h +++ b/dbms/src/DataTypes/DataTypeString.h @@ -18,6 +18,11 @@ public: return "String"; } + const char * getFamilyName() const override + { + return "String"; + } + DataTypePtr clone() const override { return std::make_shared(); diff --git a/dbms/src/DataTypes/DataTypeTuple.cpp b/dbms/src/DataTypes/DataTypeTuple.cpp index 3271e609e09..56bd4277470 100644 --- a/dbms/src/DataTypes/DataTypeTuple.cpp +++ b/dbms/src/DataTypes/DataTypeTuple.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include @@ -12,6 +14,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int EMPTY_DATA_PASSED; +} + + std::string DataTypeTuple::getName() const { std::stringstream s; @@ -253,4 +261,25 @@ Field DataTypeTuple::getDefault() const return Tuple(ext::map(elems, [] (const DataTypePtr & elem) { return elem->getDefault(); })); } + +static DataTypePtr create(const ASTPtr & arguments) +{ + if (arguments->children.empty()) + throw Exception("Tuple cannot be empty", ErrorCodes::EMPTY_DATA_PASSED); + + DataTypes nested_types; + nested_types.reserve(arguments->children.size()); + + for (const ASTPtr & child : arguments->children) + nested_types.emplace_back(DataTypeFactory::instance().get(child)); + + return std::make_shared(nested_types); +} + + +void registerDataTypeTuple(DataTypeFactory & factory) +{ + factory.registerDataType("Tuple", create); +} + } diff --git a/dbms/src/DataTypes/DataTypeTuple.h b/dbms/src/DataTypes/DataTypeTuple.h index bf9d279faf7..2b1c2356c83 100644 --- a/dbms/src/DataTypes/DataTypeTuple.h +++ b/dbms/src/DataTypes/DataTypeTuple.h @@ -19,8 +19,11 @@ public: DataTypeTuple(DataTypes elems_) : elems(elems_) {} std::string getName() const override; + const char * getFamilyName() const override { return "Tuple"; } DataTypePtr clone() const override { return std::make_shared(elems); } + bool canBeInsideNullable() const override { return false; } + void serializeBinary(const Field & field, WriteBuffer & ostr) const override; void deserializeBinary(Field & field, ReadBuffer & istr) const override; void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; diff --git a/dbms/src/DataTypes/DataTypeUUID.cpp b/dbms/src/DataTypes/DataTypeUUID.cpp index 6afe93415f0..a01d1a3132e 100644 --- a/dbms/src/DataTypes/DataTypeUUID.cpp +++ b/dbms/src/DataTypes/DataTypeUUID.cpp @@ -1,72 +1,82 @@ #include +#include + namespace DB { - void DataTypeUUID::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const - { - writeText(UUID(static_cast(column).getData()[row_num]), ostr); - } - static void deserializeText(IColumn & column, ReadBuffer & istr) - { - UUID x; - readText(x, istr); - static_cast(column).getData().push_back(x); - } - - void DataTypeUUID::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const - { - serializeText(column, row_num, ostr); - } - - void DataTypeUUID::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const - { - deserializeText(column, istr); - } - - void DataTypeUUID::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const - { - writeChar('\'', ostr); - serializeText(column, row_num, ostr); - writeChar('\'', ostr); - } - - void DataTypeUUID::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const - { - UUID x; - assertChar('\'', istr); - readText(x, istr); - assertChar('\'', istr); - static_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. - } - - void DataTypeUUID::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const - { - writeChar('"', ostr); - serializeText(column, row_num, ostr); - writeChar('"', ostr); - } - - void DataTypeUUID::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const - { - UUID x; - assertChar('"', istr); - readText(x, istr); - assertChar('"', istr); - static_cast(column).getData().push_back(x); - } - - void DataTypeUUID::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const - { - writeChar('"', ostr); - serializeText(column, row_num, ostr); - writeChar('"', ostr); - } - - void DataTypeUUID::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const - { - UUID value; - readCSV(value, istr); - static_cast(column).getData().push_back(value); - } +void DataTypeUUID::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const +{ + writeText(UUID(static_cast(column).getData()[row_num]), ostr); +} + +static void deserializeText(IColumn & column, ReadBuffer & istr) +{ + UUID x; + readText(x, istr); + static_cast(column).getData().push_back(x); +} + +void DataTypeUUID::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const +{ + serializeText(column, row_num, ostr); +} + +void DataTypeUUID::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const +{ + deserializeText(column, istr); +} + +void DataTypeUUID::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const +{ + writeChar('\'', ostr); + serializeText(column, row_num, ostr); + writeChar('\'', ostr); +} + +void DataTypeUUID::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const +{ + UUID x; + assertChar('\'', istr); + readText(x, istr); + assertChar('\'', istr); + static_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. +} + +void DataTypeUUID::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const +{ + writeChar('"', ostr); + serializeText(column, row_num, ostr); + writeChar('"', ostr); +} + +void DataTypeUUID::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const +{ + UUID x; + assertChar('"', istr); + readText(x, istr); + assertChar('"', istr); + static_cast(column).getData().push_back(x); +} + +void DataTypeUUID::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const +{ + writeChar('"', ostr); + serializeText(column, row_num, ostr); + writeChar('"', ostr); +} + +void DataTypeUUID::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const +{ + UUID value; + readCSV(value, istr); + static_cast(column).getData().push_back(value); +} + + +void registerDataTypeUUID(DataTypeFactory & factory) +{ + factory.registerSimpleDataType("UUID", [] { return DataTypePtr(std::make_shared()); }); +} + } diff --git a/dbms/src/DataTypes/DataTypeUUID.h b/dbms/src/DataTypes/DataTypeUUID.h index f7f726cd137..737da7d31bc 100644 --- a/dbms/src/DataTypes/DataTypeUUID.h +++ b/dbms/src/DataTypes/DataTypeUUID.h @@ -16,6 +16,7 @@ public: bool behavesAsNumber() const override { return false; } std::string getName() const override { return "UUID"; } + const char * getFamilyName() const override { return "UUID"; } DataTypePtr clone() const override { return std::make_shared(); } void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; diff --git a/dbms/src/DataTypes/DataTypesNumber.cpp b/dbms/src/DataTypes/DataTypesNumber.cpp new file mode 100644 index 00000000000..9ef96114052 --- /dev/null +++ b/dbms/src/DataTypes/DataTypesNumber.cpp @@ -0,0 +1,32 @@ +#include +#include + + +namespace DB +{ + +void registerDataTypeNumbers(DataTypeFactory & factory) +{ + factory.registerSimpleDataType("UInt8", [] { return DataTypePtr(std::make_shared()); }); + factory.registerSimpleDataType("UInt16", [] { return DataTypePtr(std::make_shared()); }); + factory.registerSimpleDataType("UInt32", [] { return DataTypePtr(std::make_shared()); }); + factory.registerSimpleDataType("UInt64", [] { return DataTypePtr(std::make_shared()); }); + factory.registerSimpleDataType("UInt128", [] { return DataTypePtr(std::make_shared()); }); + + factory.registerSimpleDataType("Int8", [] { return DataTypePtr(std::make_shared()); }); + factory.registerSimpleDataType("Int16", [] { return DataTypePtr(std::make_shared()); }); + factory.registerSimpleDataType("Int32", [] { return DataTypePtr(std::make_shared()); }); + factory.registerSimpleDataType("Int64", [] { return DataTypePtr(std::make_shared()); }); + + factory.registerSimpleDataType("Float32", [] { return DataTypePtr(std::make_shared()); }); + factory.registerSimpleDataType("Float64", [] { return DataTypePtr(std::make_shared()); }); + + /// These synonims are added for compatibility. + + factory.registerSimpleDataType("FLOAT", [] { return DataTypePtr(std::make_shared()); }, DataTypeFactory::CaseInsensitive); + factory.registerSimpleDataType("DOUBLE", [] { return DataTypePtr(std::make_shared()); }, DataTypeFactory::CaseInsensitive); + factory.registerSimpleDataType("INT", [] { return DataTypePtr(std::make_shared()); }, DataTypeFactory::CaseInsensitive); + factory.registerSimpleDataType("INTEGER", [] { return DataTypePtr(std::make_shared()); }, DataTypeFactory::CaseInsensitive); +} + +} diff --git a/dbms/src/DataTypes/DataTypesNumber.h b/dbms/src/DataTypes/DataTypesNumber.h index dd8db19ddf8..8c758635c26 100644 --- a/dbms/src/DataTypes/DataTypesNumber.h +++ b/dbms/src/DataTypes/DataTypesNumber.h @@ -35,6 +35,7 @@ public: using FieldType = void; std::string getName() const override { return "Void"; } + const char * getFamilyName() const override { return "Void"; } DataTypePtr clone() const override { return std::make_shared>(); } }; @@ -47,6 +48,7 @@ public: using FieldType = Null; std::string getName() const override { return "Null"; } + const char * getFamilyName() const override { return "Null"; } DataTypePtr clone() const override { return std::make_shared>(); } }; diff --git a/dbms/src/DataTypes/IDataType.h b/dbms/src/DataTypes/IDataType.h index b10fa3b0b45..4880179ca18 100644 --- a/dbms/src/DataTypes/IDataType.h +++ b/dbms/src/DataTypes/IDataType.h @@ -23,12 +23,17 @@ using DataTypes = std::vector; /** Properties of data type. * Contains methods for serialization/deserialization. + * Implementations of this interface represent a data type (example: UInt8) + * or parapetric family of data types (example: Array(...)). */ class IDataType { public: /// Name of data type (examples: UInt64, Array(String)). - virtual std::string getName() const = 0; + virtual String getName() const = 0; + + /// Name of data type family (example: FixedString, Array). + virtual const char * getFamilyName() const = 0; /// Is this type the null type? TODO Move this method to separate "traits" classes. virtual bool isNull() const { return false; } @@ -49,6 +54,9 @@ public: /// If this data type cannot appear in table declaration - only for intermediate values of calculations. virtual bool notForTables() const { return false; } + /// If this data type cannot be wrapped in Nullable data type. + virtual bool canBeInsideNullable() const { return true; } + virtual DataTypePtr clone() const = 0; /** Binary serialization for range of values in column - for writing to disk/network, etc. diff --git a/dbms/src/DataTypes/IDataTypeDummy.h b/dbms/src/DataTypes/IDataTypeDummy.h index 7f7d9b7a63f..5ee1242d095 100644 --- a/dbms/src/DataTypes/IDataTypeDummy.h +++ b/dbms/src/DataTypes/IDataTypeDummy.h @@ -19,6 +19,11 @@ private: return true; } + bool canBeInsideNullable() const override + { + return false; + } + void throwNoSerialization() const { throw Exception("Serialization is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Functions/FunctionsArithmetic.h b/dbms/src/Functions/FunctionsArithmetic.h index e4ecdd06602..6c3cda38862 100644 --- a/dbms/src/Functions/FunctionsArithmetic.h +++ b/dbms/src/Functions/FunctionsArithmetic.h @@ -616,8 +616,8 @@ private: bool executeRightTypeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, const ColumnType * col_left) { - throw Exception("Types " + TypeName::get() - + " and " + TypeName::get() + throw Exception("Types " + String(TypeName::get()) + + " and " + String(TypeName::get()) + " are incompatible for function " + getName() + " or not upscaleable to common type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } diff --git a/dbms/src/Functions/FunctionsArray.h b/dbms/src/Functions/FunctionsArray.h index 1303caed2ff..2f9eacff177 100644 --- a/dbms/src/Functions/FunctionsArray.h +++ b/dbms/src/Functions/FunctionsArray.h @@ -1303,7 +1303,7 @@ private: }; template -const String FunctionEmptyArray::name = FunctionEmptyArray::base_name + DataTypeToName::get(); +const String FunctionEmptyArray::name = FunctionEmptyArray::base_name + String(DataTypeToName::get()); class FunctionRange : public IFunction { diff --git a/dbms/src/Functions/FunctionsEmbeddedDictionaries.h b/dbms/src/Functions/FunctionsEmbeddedDictionaries.h index d3ccf11073b..72ebfd29cb9 100644 --- a/dbms/src/Functions/FunctionsEmbeddedDictionaries.h +++ b/dbms/src/Functions/FunctionsEmbeddedDictionaries.h @@ -202,12 +202,12 @@ public: if (arguments[0]->getName() != TypeName::get()) throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName() - + " (must be " + TypeName::get() + ")", + + " (must be " + String(TypeName::get()) + ")", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (arguments.size() == 2 && arguments[1]->getName() != TypeName::get()) throw Exception("Illegal type " + arguments[1]->getName() + " of the second ('point of view') argument of function " + getName() - + " (must be " + TypeName::get() + ")", + + " (must be " + String(TypeName::get()) + ")", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return arguments[0]; @@ -295,17 +295,17 @@ public: if (arguments[0]->getName() != TypeName::get()) throw Exception("Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() - + " (must be " + TypeName::get() + ")", + + " (must be " + String(TypeName::get()) + ")", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (arguments[1]->getName() != TypeName::get()) throw Exception("Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() - + " (must be " + TypeName::get() + ")", + + " (must be " + String(TypeName::get()) + ")", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (arguments.size() == 3 && arguments[2]->getName() != TypeName::get()) throw Exception("Illegal type " + arguments[2]->getName() + " of the third ('point of view') argument of function " + getName() - + " (must be " + TypeName::get() + ")", + + " (must be " + String(TypeName::get()) + ")", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared(); @@ -428,12 +428,12 @@ public: if (arguments[0]->getName() != TypeName::get()) throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName() - + " (must be " + TypeName::get() + ")", + + " (must be " + String(TypeName::get()) + ")", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (arguments.size() == 2 && arguments[1]->getName() != TypeName::get()) throw Exception("Illegal type " + arguments[1]->getName() + " of the second ('point of view') argument of function " + getName() - + " (must be " + TypeName::get() + ")", + + " (must be " + String(TypeName::get()) + ")", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared(arguments[0]); @@ -714,12 +714,12 @@ public: if (arguments[0]->getName() != TypeName::get()) throw Exception("Illegal type " + arguments[0]->getName() + " of the first argument of function " + getName() - + " (must be " + TypeName::get() + ")", + + " (must be " + String(TypeName::get()) + ")", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (arguments.size() == 2 && arguments[1]->getName() != TypeName::get()) throw Exception("Illegal type " + arguments[0]->getName() + " of the second argument of function " + getName() - + " (must be " + TypeName::get() + ")", + + " (must be " + String(TypeName::get()) + ")", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared(); diff --git a/dbms/src/Functions/FunctionsTransform.cpp b/dbms/src/Functions/FunctionsTransform.cpp index 6166e9a7f75..f338078823e 100644 --- a/dbms/src/Functions/FunctionsTransform.cpp +++ b/dbms/src/Functions/FunctionsTransform.cpp @@ -19,7 +19,7 @@ struct TypeProcessorImpl auto type_res = DataTypeTraits::ToEnrichedDataTypeObject::execute(); if ((type_res.first == DataTypePtr()) && (type_res.second == DataTypePtr())) - throw Exception("Types " + TypeName::get() + " and " + TypeName::get() + throw Exception("Types " + String(TypeName::get()) + " and " + String(TypeName::get()) + " are not upscalable to a common type without loss of precision", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return type_res; @@ -31,16 +31,16 @@ struct RightTypeProcessor { static DataTypeTraits::EnrichedDataTypePtr execute(const IDataType & type2) { - if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); - if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); - if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); - if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); - if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); - if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); - if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); - if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); - if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); - if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); + if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); + if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); + if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); + if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); + if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); + if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); + if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); + if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); + if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); + if (typeid_cast(&type2)) return TypeProcessorImpl::execute(); throw Exception("Logical error: not a numeric type passed to function getSmallestCommonNumericType", ErrorCodes::LOGICAL_ERROR); } @@ -51,17 +51,17 @@ struct LeftTypeProcessor { static DataTypeTraits::EnrichedDataTypePtr execute(const DataTypePtr & right, const IDataType & type2) { - if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); - if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); - if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); - if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); - if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); - if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); - if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); - if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); - if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); - if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); - if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); + if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); + if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); + if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); + if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); + if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); + if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); + if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); + if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); + if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); + if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); + if (typeid_cast(&*right)) return RightTypeProcessor::execute(type2); throw Exception("Logical error: not a numeric type passed to function getSmallestCommonNumericType", ErrorCodes::LOGICAL_ERROR); } @@ -74,16 +74,16 @@ DataTypeTraits::EnrichedDataTypePtr getSmallestCommonNumericType(const DataTypeT const DataTypePtr & left = type1.first; const DataTypePtr & right = type1.second; - if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); - if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); - if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); - if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); - if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); - if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); - if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); - if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); - if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); - if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); + if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); + if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); + if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); + if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); + if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); + if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); + if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); + if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); + if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); + if (typeid_cast(&*left)) return LeftTypeProcessor::execute(right, type2); throw Exception("Logical error: not a numeric type passed to function getSmallestCommonNumericType", ErrorCodes::LOGICAL_ERROR); } diff --git a/dbms/src/Functions/FunctionsURL.h b/dbms/src/Functions/FunctionsURL.h index 1551ddf30f1..ffa7b48f291 100644 --- a/dbms/src/Functions/FunctionsURL.h +++ b/dbms/src/Functions/FunctionsURL.h @@ -2,7 +2,6 @@ #include #include -#include #include #include #include diff --git a/dbms/src/Functions/FunctionsVisitParam.h b/dbms/src/Functions/FunctionsVisitParam.h index 6e71d60b901..86512a4c841 100644 --- a/dbms/src/Functions/FunctionsVisitParam.h +++ b/dbms/src/Functions/FunctionsVisitParam.h @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/dbms/src/Functions/IFunction.cpp b/dbms/src/Functions/IFunction.cpp index 8a8c042ea09..946cc422259 100644 --- a/dbms/src/Functions/IFunction.cpp +++ b/dbms/src/Functions/IFunction.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include diff --git a/dbms/src/Functions/tests/logical_functions_performance.cpp b/dbms/src/Functions/tests/logical_functions_performance.cpp index a8a37e04cde..0d627848e38 100644 --- a/dbms/src/Functions/tests/logical_functions_performance.cpp +++ b/dbms/src/Functions/tests/logical_functions_performance.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 871507cfdba..e204dda9c16 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -192,7 +191,7 @@ void ExpressionAnalyzer::init() /// Executing scalar subqueries - replacing them with constant values. executeScalarSubqueries(); - /// Optimize if with constant condition after constats are substituted instead of sclalar subqueries + /// Optimize if with constant condition after constants was substituted instead of sclalar subqueries. optimizeIfWithConstantCondition(); /// GROUP BY injective function elimination. @@ -624,8 +623,118 @@ void ExpressionAnalyzer::findExternalTables(ASTPtr & ast) } +static std::pair getDatabaseAndTableNameFromIdentifier(const ASTIdentifier & identifier) +{ + std::pair res; + res.second = identifier.name; + if (!identifier.children.empty()) + { + if (identifier.children.size() != 2) + throw Exception("Qualified table name could have only two components", ErrorCodes::LOGICAL_ERROR); + + res.first = typeid_cast(*identifier.children[0]).name; + res.second = typeid_cast(*identifier.children[1]).name; + } + return res; +} + + static std::shared_ptr interpretSubquery( - ASTPtr & subquery_or_table_name, const Context & context, size_t subquery_depth, const Names & required_columns); + const ASTPtr & subquery_or_table_name, const Context & context, size_t subquery_depth, const Names & required_columns) +{ + /// Subquery or table name. The name of the table is similar to the subquery `SELECT * FROM t`. + const ASTSubquery * subquery = typeid_cast(subquery_or_table_name.get()); + const ASTIdentifier * table = typeid_cast(subquery_or_table_name.get()); + + if (!subquery && !table) + throw Exception("IN/JOIN supports only SELECT subqueries.", ErrorCodes::BAD_ARGUMENTS); + + /** The subquery in the IN / JOIN section does not have any restrictions on the maximum size of the result. + * Because the result of this query is not the result of the entire query. + * Constraints work instead + * max_rows_in_set, max_bytes_in_set, set_overflow_mode, + * max_rows_in_join, max_bytes_in_join, join_overflow_mode, + * which are checked separately (in the Set, Join objects). + */ + Context subquery_context = context; + Settings subquery_settings = context.getSettings(); + subquery_settings.limits.max_result_rows = 0; + subquery_settings.limits.max_result_bytes = 0; + /// The calculation of `extremes` does not make sense and is not necessary (if you do it, then the `extremes` of the subquery can be taken instead of the whole query). + subquery_settings.extremes = 0; + subquery_context.setSettings(subquery_settings); + + ASTPtr query; + if (table) + { + /// create ASTSelectQuery for "SELECT * FROM table" as if written by hand + const auto select_query = std::make_shared(); + query = select_query; + + const auto select_expression_list = std::make_shared(); + select_query->select_expression_list = select_expression_list; + select_query->children.emplace_back(select_query->select_expression_list); + + /// get columns list for target table + auto database_table = getDatabaseAndTableNameFromIdentifier(*table); + const auto & storage = context.getTable(database_table.first, database_table.second); + const auto & columns = storage->getColumnsListNonMaterialized(); + select_expression_list->children.reserve(columns.size()); + + /// manually substitute column names in place of asterisk + for (const auto & column : columns) + select_expression_list->children.emplace_back(std::make_shared( + StringRange{}, column.name)); + + select_query->replaceDatabaseAndTable(database_table.first, database_table.second); + } + else + { + query = subquery->children.at(0); + + /** Columns with the same name can be specified in a subquery. For example, SELECT x, x FROM t + * This is bad, because the result of such a query can not be saved to the table, because the table can not have the same name columns. + * Saving to the table is required for GLOBAL subqueries. + * + * To avoid this situation, we will rename the same columns. + */ + + std::set all_column_names; + std::set assigned_column_names; + + if (ASTSelectQuery * select = typeid_cast(query.get())) + { + for (auto & expr : select->select_expression_list->children) + all_column_names.insert(expr->getAliasOrColumnName()); + + for (auto & expr : select->select_expression_list->children) + { + auto name = expr->getAliasOrColumnName(); + + if (!assigned_column_names.insert(name).second) + { + size_t i = 1; + while (all_column_names.end() != all_column_names.find(name + "_" + toString(i))) + ++i; + + name = name + "_" + toString(i); + expr = expr->clone(); /// Cancels fuse of the same expressions in the tree. + expr->setAlias(name); + + all_column_names.insert(name); + assigned_column_names.insert(name); + } + } + } + } + + if (required_columns.empty()) + return std::make_shared( + query, subquery_context, QueryProcessingStage::Complete, subquery_depth + 1); + else + return std::make_shared( + query, subquery_context, required_columns, QueryProcessingStage::Complete, subquery_depth + 1); +} void ExpressionAnalyzer::addExternalStorage(ASTPtr & subquery_or_table_name_or_table_expression) @@ -1381,18 +1490,19 @@ void ExpressionAnalyzer::makeSetsForIndex() makeSetsForIndexImpl(ast, storage->getSampleBlock()); } -void ExpressionAnalyzer::makeSetsForIndexImpl(ASTPtr & node, const Block & sample_block) +void ExpressionAnalyzer::makeSetsForIndexImpl(const ASTPtr & node, const Block & sample_block) { for (auto & child : node->children) makeSetsForIndexImpl(child, sample_block); - ASTFunction * func = typeid_cast(node.get()); + const ASTFunction * func = typeid_cast(node.get()); if (func && func->kind == ASTFunction::FUNCTION && functionIsInOperator(func->name)) { - IAST & args = *func->arguments; - ASTPtr & arg = args.children.at(1); + const IAST & args = *func->arguments; + const ASTPtr & arg = args.children.at(1); - if (!typeid_cast(arg.get()) && !typeid_cast(arg.get()) && !typeid_cast(arg.get())) + if (!prepared_sets.count(arg.get()) /// Not already prepared. + && !typeid_cast(arg.get()) && !typeid_cast(arg.get())) { try { @@ -1409,141 +1519,25 @@ void ExpressionAnalyzer::makeSetsForIndexImpl(ASTPtr & node, const Block & sampl } -static std::pair getDatabaseAndTableNameFromIdentifier(const ASTIdentifier & identifier) -{ - std::pair res; - res.second = identifier.name; - if (!identifier.children.empty()) - { - if (identifier.children.size() != 2) - throw Exception("Qualified table name could have only two components", ErrorCodes::LOGICAL_ERROR); - - res.first = typeid_cast(*identifier.children[0]).name; - res.second = typeid_cast(*identifier.children[1]).name; - } - return res; -} - - -static std::shared_ptr interpretSubquery( - ASTPtr & subquery_or_table_name, const Context & context, size_t subquery_depth, const Names & required_columns) -{ - /// Subquery or table name. The name of the table is similar to the subquery `SELECT * FROM t`. - const ASTSubquery * subquery = typeid_cast(subquery_or_table_name.get()); - const ASTIdentifier * table = typeid_cast(subquery_or_table_name.get()); - - if (!subquery && !table) - throw Exception("IN/JOIN supports only SELECT subqueries.", ErrorCodes::BAD_ARGUMENTS); - - /** The subquery in the IN / JOIN section does not have any restrictions on the maximum size of the result. - * Because the result of this query is not the result of the entire query. - * Constraints work instead - * max_rows_in_set, max_bytes_in_set, set_overflow_mode, - * max_rows_in_join, max_bytes_in_join, join_overflow_mode, - * which are checked separately (in the Set, Join objects). - */ - Context subquery_context = context; - Settings subquery_settings = context.getSettings(); - subquery_settings.limits.max_result_rows = 0; - subquery_settings.limits.max_result_bytes = 0; - /// The calculation of `extremes` does not make sense and is not necessary (if you do it, then the `extremes` of the subquery can be taken instead of the whole query). - subquery_settings.extremes = 0; - subquery_context.setSettings(subquery_settings); - - ASTPtr query; - if (table) - { - /// create ASTSelectQuery for "SELECT * FROM table" as if written by hand - const auto select_query = std::make_shared(); - query = select_query; - - const auto select_expression_list = std::make_shared(); - select_query->select_expression_list = select_expression_list; - select_query->children.emplace_back(select_query->select_expression_list); - - /// get columns list for target table - auto database_table = getDatabaseAndTableNameFromIdentifier(*table); - const auto & storage = context.getTable(database_table.first, database_table.second); - const auto & columns = storage->getColumnsListNonMaterialized(); - select_expression_list->children.reserve(columns.size()); - - /// manually substitute column names in place of asterisk - for (const auto & column : columns) - select_expression_list->children.emplace_back(std::make_shared( - StringRange{}, column.name)); - - select_query->replaceDatabaseAndTable(database_table.first, database_table.second); - } - else - { - query = subquery->children.at(0); - - /** Columns with the same name can be specified in a subquery. For example, SELECT x, x FROM t - * This is bad, because the result of such a query can not be saved to the table, because the table can not have the same name columns. - * Saving to the table is required for GLOBAL subqueries. - * - * To avoid this situation, we will rename the same columns. - */ - - std::set all_column_names; - std::set assigned_column_names; - - if (ASTSelectQuery * select = typeid_cast(query.get())) - { - for (auto & expr : select->select_expression_list->children) - all_column_names.insert(expr->getAliasOrColumnName()); - - for (auto & expr : select->select_expression_list->children) - { - auto name = expr->getAliasOrColumnName(); - - if (!assigned_column_names.insert(name).second) - { - size_t i = 1; - while (all_column_names.end() != all_column_names.find(name + "_" + toString(i))) - ++i; - - name = name + "_" + toString(i); - expr = expr->clone(); /// Cancels fuse of the same expressions in the tree. - expr->setAlias(name); - - all_column_names.insert(name); - assigned_column_names.insert(name); - } - } - } - } - - if (required_columns.empty()) - return std::make_shared( - query, subquery_context, QueryProcessingStage::Complete, subquery_depth + 1); - else - return std::make_shared( - query, subquery_context, required_columns, QueryProcessingStage::Complete, subquery_depth + 1); -} - - -void ExpressionAnalyzer::makeSet(ASTFunction * node, const Block & sample_block) +void ExpressionAnalyzer::makeSet(const ASTFunction * node, const Block & sample_block) { /** You need to convert the right argument to a set. * This can be a table name, a value, a value enumeration, or a subquery. * The enumeration of values is parsed as a function `tuple`. */ - IAST & args = *node->arguments; - ASTPtr & arg = args.children.at(1); + const IAST & args = *node->arguments; + const ASTPtr & arg = args.children.at(1); /// Already converted. - if (typeid_cast(arg.get())) + if (prepared_sets.count(arg.get())) return; /// If the subquery or table name for SELECT. - ASTIdentifier * identifier = typeid_cast(arg.get()); - if (typeid_cast(arg.get()) || identifier) + const ASTIdentifier * identifier = typeid_cast(arg.get()); + if (typeid_cast(arg.get()) || identifier) { /// We get the stream of blocks for the subquery. Create Set and put it in place of the subquery. String set_id = arg->getColumnName(); - auto ast_set = std::make_shared(set_id); - ASTPtr ast_set_ptr = ast_set; /// A special case is if the name of the table is specified on the right side of the IN statement, /// and the table has the type Set (a previously prepared set). @@ -1558,9 +1552,7 @@ void ExpressionAnalyzer::makeSet(ASTFunction * node, const Block & sample_block) if (storage_set) { - SetPtr & set = storage_set->getSet(); - ast_set->set = set; - arg = ast_set_ptr; + prepared_sets[arg.get()] = storage_set->getSet(); return; } } @@ -1571,12 +1563,11 @@ void ExpressionAnalyzer::makeSet(ASTFunction * node, const Block & sample_block) /// If you already created a Set with the same subquery / table. if (subquery_for_set.set) { - ast_set->set = subquery_for_set.set; - arg = ast_set_ptr; + prepared_sets[arg.get()] = subquery_for_set.set; return; } - ast_set->set = std::make_shared(settings.limits); + SetPtr set = std::make_shared(settings.limits); /** The following happens for GLOBAL INs: * - in the addExternalStorage function, the IN (SELECT ...) subquery is replaced with IN _data1, @@ -1618,8 +1609,8 @@ void ExpressionAnalyzer::makeSet(ASTFunction * node, const Block & sample_block) */ } - subquery_for_set.set = ast_set->set; - arg = ast_set_ptr; + subquery_for_set.set = set; + prepared_sets[arg.get()] = set; } else { @@ -1629,23 +1620,23 @@ void ExpressionAnalyzer::makeSet(ASTFunction * node, const Block & sample_block) } /// The case of an explicit enumeration of values. -void ExpressionAnalyzer::makeExplicitSet(ASTFunction * node, const Block & sample_block, bool create_ordered_set) +void ExpressionAnalyzer::makeExplicitSet(const ASTFunction * node, const Block & sample_block, bool create_ordered_set) { - IAST & args = *node->arguments; + const IAST & args = *node->arguments; if (args.children.size() != 2) throw Exception("Wrong number of arguments passed to function in", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - ASTPtr & arg = args.children.at(1); + const ASTPtr & arg = args.children.at(1); DataTypes set_element_types; - ASTPtr & left_arg = args.children.at(0); + const ASTPtr & left_arg = args.children.at(0); - ASTFunction * left_arg_tuple = typeid_cast(left_arg.get()); + const ASTFunction * left_arg_tuple = typeid_cast(left_arg.get()); /** NOTE If tuple in left hand side specified non-explicitly * Example: identity((a, b)) IN ((1, 2), (3, 4)) - * instead of (a, b)) IN ((1, 2), (3, 4)) + * instead of (a, b)) IN ((1, 2), (3, 4)) * then set creation of set doesn't work correctly. */ if (left_arg_tuple && left_arg_tuple->name == "tuple") @@ -1654,7 +1645,7 @@ void ExpressionAnalyzer::makeExplicitSet(ASTFunction * node, const Block & sampl { const auto & data_type = sample_block.getByName(arg->getColumnName()).type; - /// @note prevent crash in query: SELECT (1, [1]) in (1, 1) + /// NOTE prevent crash in query: SELECT (1, [1]) in (1, 1) if (const auto array = typeid_cast(data_type.get())) throw Exception("Incorrect element of tuple: " + array->getName(), ErrorCodes::INCORRECT_ELEMENT_OF_SET); @@ -1720,11 +1711,9 @@ void ExpressionAnalyzer::makeExplicitSet(ASTFunction * node, const Block & sampl elements_ast = exp_list; } - auto ast_set = std::make_shared(arg->getColumnName()); - ast_set->set = std::make_shared(settings.limits); - ast_set->is_explicit = true; - ast_set->set->createFromAST(set_element_types, elements_ast, context, create_ordered_set); - arg = ast_set; + SetPtr set = std::make_shared(settings.limits); + set->createFromAST(set_element_types, elements_ast, context, create_ordered_set); + prepared_sets[arg.get()] = std::move(set); } @@ -2057,7 +2046,6 @@ void ExpressionAnalyzer::getActionsImpl(ASTPtr ast, bool no_subqueries, bool onl for (auto & child : node->arguments->children) { ASTFunction * lambda = typeid_cast(child.get()); - ASTSet * set = typeid_cast(child.get()); if (lambda && lambda->name == "lambda") { /// If the argument is a lambda expression, just remember its approximate type. @@ -2074,21 +2062,23 @@ void ExpressionAnalyzer::getActionsImpl(ASTPtr ast, bool no_subqueries, bool onl /// Select the name in the next cycle. argument_names.emplace_back(); } - else if (set) + else if (prepared_sets.count(child.get())) { ColumnWithTypeAndName column; column.type = std::make_shared(); - /// If the argument is a set given by an enumeration of values, give it a unique name, + const SetPtr & set = prepared_sets[child.get()]; + + /// If the argument is a set given by an enumeration of values (so, the set was already built), give it a unique name, /// so that sets with the same record do not fuse together (they can have different types). - if (set->is_explicit) + if (!set->empty()) column.name = getUniqueName(actions_stack.getSampleBlock(), "__set"); else - column.name = set->getColumnName(); + column.name = child->getColumnName(); if (!actions_stack.getSampleBlock().has(column.name)) { - column.column = std::make_shared(1, set->set); + column.column = std::make_shared(1, set); actions_stack.addAction(ExpressionAction::addColumn(column)); } diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.h b/dbms/src/Interpreters/ExpressionAnalyzer.h index c9e46d04b14..44b5b6d78ee 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.h +++ b/dbms/src/Interpreters/ExpressionAnalyzer.h @@ -13,15 +13,16 @@ class Context; class ExpressionActions; struct ExpressionActionsChain; -class Set; -using SetPtr = std::shared_ptr; - class Join; using JoinPtr = std::shared_ptr; class IAST; using ASTPtr = std::shared_ptr; +class Set; +using SetPtr = std::shared_ptr; +using PreparedSets = std::unordered_map; + class IBlockInputStream; using BlockInputStreamPtr = std::shared_ptr; @@ -42,10 +43,8 @@ struct SubqueryForSet BlockInputStreamPtr source; Block source_sample; - /// If set, create from Set result. + /// If set, build it from result. SetPtr set; - - /// If set, create from Join result. JoinPtr join; /// If set, put the result into the table. @@ -130,6 +129,8 @@ public: */ SubqueriesForSets getSubqueriesForSets() { return subqueries_for_sets; } + PreparedSets getPreparedSets() { return prepared_sets; } + /** Tables that will need to be sent to remote servers for distributed query processing. */ const Tables & getExternalTables() const { return external_tables; } @@ -167,6 +168,8 @@ private: SubqueriesForSets subqueries_for_sets; + PreparedSets prepared_sets; + /// NOTE: So far, only one JOIN per query is supported. /** Query of the form `SELECT expr(x) AS FROM t1 ANY LEFT JOIN (SELECT expr(x) AS k FROM t2) USING k` @@ -244,8 +247,7 @@ private: void optimizeIfWithConstantConditionImpl(ASTPtr & current_ast, Aliases & aliases) const; bool tryExtractConstValueFromCondition(const ASTPtr & condition, bool & value) const; - /// Transform the value enumeration or subquery into ASTSet. `node` - `in` or `notIn` function. - void makeSet(ASTFunction * node, const Block & sample_block); + void makeSet(const ASTFunction * node, const Block & sample_block); /// Adds a list of ALIAS columns from the table void addAliasColumns(); @@ -309,8 +311,8 @@ private: /** Create Set from an explicit enumeration of values in the query. * If create_ordered_set = true - create a data structure suitable for using the index. */ - void makeExplicitSet(ASTFunction * node, const Block & sample_block, bool create_ordered_set); - void makeSetsForIndexImpl(ASTPtr & node, const Block & sample_block); + void makeExplicitSet(const ASTFunction * node, const Block & sample_block, bool create_ordered_set); + void makeSetsForIndexImpl(const ASTPtr & node, const Block & sample_block); /** Translate qualified names such as db.table.column, table.column, table_alias.column * to unqualified names. This is done in a poor transitional way: diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index d6c7be640a7..2834fb4636a 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -30,6 +30,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int ARGUMENT_OUT_OF_BOUND; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; } @@ -53,6 +54,7 @@ BlockIO InterpreterAlterQuery::execute() PartitionCommands partition_commands; parseAlter(alter.parameters, alter_commands, partition_commands); + partition_commands.validate(table.get()); for (const PartitionCommand & command : partition_commands) { switch (command.type) @@ -89,7 +91,6 @@ BlockIO InterpreterAlterQuery::execute() return {}; alter_commands.validate(table.get(), context); - table->alter(alter_commands, database_name, table_name, context); return {}; @@ -230,4 +231,23 @@ void InterpreterAlterQuery::parseAlter( } } + +void InterpreterAlterQuery::PartitionCommands::validate(const IStorage * table) +{ + for (const PartitionCommand & command : *this) + { + if (command.type == PartitionCommand::CLEAR_COLUMN) + { + String column_name = command.column_name.safeGet(); + + if (!table->hasRealColumn(column_name)) + { + throw Exception("Wrong column name. Cannot find column " + column_name + " to clear it from partition", + DB::ErrorCodes::ILLEGAL_COLUMN); + } + } + } +} + + } diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.h b/dbms/src/Interpreters/InterpreterAlterQuery.h index 41a0f8a318b..753037072c7 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.h +++ b/dbms/src/Interpreters/InterpreterAlterQuery.h @@ -112,7 +112,11 @@ private: } }; - using PartitionCommands = std::vector; + class PartitionCommands : public std::vector + { + public: + void validate(const IStorage * table); + }; ASTPtr query_ptr; diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 919f2f7fd2e..6ea6e6600b2 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -178,17 +178,13 @@ static ColumnsAndDefaults parseColumns( ASTPtr default_expr_list = std::make_shared(); default_expr_list->children.reserve(column_list_ast.children.size()); - const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); - for (auto & ast : column_list_ast.children) { auto & col_decl = typeid_cast(*ast); if (col_decl.type) { - const auto & type_range = col_decl.type->range; - columns.emplace_back(col_decl.name, - data_type_factory.get({ type_range.first, type_range.second })); + columns.emplace_back(col_decl.name, DataTypeFactory::instance().get(col_decl.type)); } else /// we're creating dummy DataTypeUInt8 in order to prevent the NullPointerException in ExpressionActions diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 9c2b9136552..caf4a2dad96 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -836,13 +836,17 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns() if (max_streams > 1 && !is_remote) max_streams *= settings.max_streams_to_max_threads_ratio; + SelectQueryInfo query_info; + query_info.query = query_ptr; + query_info.sets = query_analyzer->getPreparedSets(); + /// PREWHERE optimization { auto optimize_prewhere = [&](auto & merge_tree) { /// Try transferring some condition from WHERE to PREWHERE if enabled and viable if (settings.optimize_move_to_prewhere && query.where_expression && !query.prewhere_expression && !query.final()) - MergeTreeWhereOptimizer{query_ptr, context, merge_tree.getData(), required_columns, log}; + MergeTreeWhereOptimizer{query_info, context, merge_tree.getData(), required_columns, log}; }; if (const StorageMergeTree * merge_tree = typeid_cast(storage.get())) @@ -851,7 +855,7 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns() optimize_prewhere(*merge_tree); } - streams = storage->read(required_columns, query_ptr, context, from_stage, max_block_size, max_streams); + streams = storage->read(required_columns, query_info, context, from_stage, max_block_size, max_streams); if (alias_actions) { diff --git a/dbms/src/Interpreters/Set.h b/dbms/src/Interpreters/Set.h index 97aa6309d29..055a4ee8e90 100644 --- a/dbms/src/Interpreters/Set.h +++ b/dbms/src/Interpreters/Set.h @@ -2,7 +2,6 @@ #include #include -#include #include #include #include diff --git a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp index a34756c1d30..927f7038385 100644 --- a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp +++ b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include diff --git a/dbms/src/Parsers/ASTSet.h b/dbms/src/Parsers/ASTSet.h deleted file mode 100644 index 9f7c8c87ffb..00000000000 --- a/dbms/src/Parsers/ASTSet.h +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ - -class Set; - -/** The set. During the calculation, the expression in the IN section is replaced by the set - * - a subquery or an explicit enumeration of values. - * TODO: This is bad practice, lead to bugs. Remove it. - */ -class ASTSet : public IAST -{ -public: - std::shared_ptr set; - String column_name; - bool is_explicit = false; - - ASTSet(const String & column_name_) : column_name(column_name_) {} - ASTSet(const StringRange range_, const String & column_name_) : IAST(range_), column_name(column_name_) {} - String getID() const override { return "Set_" + getColumnName(); } - ASTPtr clone() const override { return std::make_shared(*this); } - String getColumnName() const override { return column_name; } - -protected: - void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override - { - /** Prepared set. In user requests, this does not happen, but this happens after the intermediate query transformation. - * Output it for not real (this will not be a valid query, but it will show that there was a set). - */ - settings.ostr << (settings.hilite ? hilite_keyword : "") - << "(...)" - << (settings.hilite ? hilite_none : ""); - } -}; - -} diff --git a/dbms/src/Parsers/ParserEnumElement.cpp b/dbms/src/Parsers/ParserEnumElement.cpp deleted file mode 100644 index 931f09a04cd..00000000000 --- a/dbms/src/Parsers/ParserEnumElement.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include -#include -#include -#include - - -namespace DB -{ - -bool ParserEnumElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) -{ - ParserStringLiteral name_parser; - ParserNumber value_parser; - ParserToken equality_sign_parser(TokenType::Equals); - - const auto begin = pos; - - ASTPtr name; - if (!name_parser.parse(pos, name, expected)) - return false; - - if (!equality_sign_parser.ignore(pos, expected)) - return false; - - ASTPtr value; - if (!value_parser.parse(pos, value, expected)) - return false; - - node = std::make_shared( - StringRange{begin, pos}, static_cast(*name).value.get(), static_cast(*value).value); - - return true; -} - -} diff --git a/dbms/src/Parsers/ParserEnumElement.h b/dbms/src/Parsers/ParserEnumElement.h deleted file mode 100644 index c6a86fa824f..00000000000 --- a/dbms/src/Parsers/ParserEnumElement.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include - -namespace DB -{ - -class ParserEnumElement : public IParserBase -{ -protected: - const char * getName() const override { return "Enum element"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; -}; - - -} diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index a61d262df5a..05dd03a202f 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -2,19 +2,28 @@ #include #include #include + #include #include #include -#include #include #include +#include #include + #include #include + #include #include #include #include +#include +#include +#include +#include +#include +#include #include #include #include @@ -25,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -39,13 +47,6 @@ #include #include #include "InterruptListener.h" -#include -#include -#include -#include -#include -#include -#include #include #include diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index b9c71c70462..91d54661733 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -30,10 +31,6 @@ class IStorage; using StoragePtr = std::shared_ptr; -class IAST; - -using ASTPtr = std::shared_ptr; - struct Settings; class AlterCommands; @@ -156,7 +153,7 @@ public: * (indexes, locks, etc.) * Returns a stream with which you can read data sequentially * or multiple streams for parallel data reading. - * The into `processed_stage` info is also written to what stage the request was processed. + * The `processed_stage` info is also written to what stage the request was processed. * (Normally, the function only reads the columns from the list, but in other cases, * for example, the request can be partially processed on a remote server.) * @@ -171,7 +168,7 @@ public: */ virtual BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 6a8aa1b6489..e29d4b29fe0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -129,7 +129,7 @@ static RelativeSize convertAbsoluteSampleSizeToRelative(const ASTPtr & node, siz BlockInputStreams MergeTreeDataSelectExecutor::read( const Names & column_names_to_return, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, @@ -188,7 +188,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::read( /// If `_part` virtual column is requested, we try to use it as an index. Block virtual_columns_block = getBlockWithPartColumn(parts); if (part_column_queried) - VirtualColumnUtils::filterBlockWithQuery(query, virtual_columns_block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, virtual_columns_block, context); std::multiset part_values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_part"); @@ -199,9 +199,9 @@ BlockInputStreams MergeTreeDataSelectExecutor::read( SortDescription sort_descr = data.getSortDescription(); ColumnsWithTypeAndName date_columns = {{DataTypeDate{}.createColumn(), std::make_shared(), data.date_column_name}}; - PKCondition key_condition(query, context, available_real_and_virtual_columns, sort_descr, + PKCondition key_condition(query_info, context, available_real_and_virtual_columns, sort_descr, data.getPrimaryExpression()); - PKCondition date_condition(query, context, available_real_and_virtual_columns, + PKCondition date_condition(query_info, context, available_real_and_virtual_columns, SortDescription(1, SortColumnDescription(data.date_column_name, 1, 1)), std::make_shared(date_columns, settings)); @@ -254,7 +254,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::read( RelativeSize relative_sample_size = 0; RelativeSize relative_sample_offset = 0; - ASTSelectQuery & select = *typeid_cast(&*query); + ASTSelectQuery & select = typeid_cast(*query_info.query); auto select_sample_size = select.sample_size(); auto select_sample_offset = select.sample_offset(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 37b9f23de35..1e1c48223df 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -22,7 +22,7 @@ public: */ BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 4974274e901..61e61014d29 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -35,18 +34,21 @@ static constexpr auto global_not_in_function_name = "globalNotIn"; MergeTreeWhereOptimizer::MergeTreeWhereOptimizer( - ASTPtr & query, const Context & context, const MergeTreeData & data, const Names & column_names, + SelectQueryInfo & query_info, + const Context & context, + const MergeTreeData & data, + const Names & column_names, Logger * log) - : primary_key_columns{ext::map(data.getSortDescription(), - [] (const SortColumnDescription & col) { return col.column_name; }) - }, + : primary_key_columns{ext::map(data.getSortDescription(), + [] (const SortColumnDescription & col) { return col.column_name; })}, table_columns{ext::map(data.getColumnsList(), - [] (const NameAndTypePair & col) { return col.name; }) - }, block_with_constants{PKCondition::getBlockWithConstants(query, context, data.getColumnsList())}, + [] (const NameAndTypePair & col) { return col.name; })}, + block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.getColumnsList())}, + prepared_sets(query_info.sets), log{log} { calculateColumnSizes(data, column_names); - auto & select = typeid_cast(*query); + auto & select = typeid_cast(*query_info.query); determineArrayJoinedNames(select); optimize(select); } @@ -330,7 +332,7 @@ bool MergeTreeWhereOptimizer::isPrimaryKeyAtom(const IAST * const ast) const if ((primary_key_columns.count(first_arg_name) && isConstant(args[1])) || (primary_key_columns.count(second_arg_name) && isConstant(args[0])) || (primary_key_columns.count(first_arg_name) - && (typeid_cast(args[1].get()) || typeid_cast(args[1].get())))) + && (prepared_sets.count(args[1].get()) || typeid_cast(args[1].get())))) return true; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.h b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.h index 9b99806b511..855f35565e8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.h +++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace Poco { class Logger; } @@ -12,9 +13,6 @@ namespace Poco { class Logger; } namespace DB { -class IAST; -using ASTPtr = std::shared_ptr; - class ASTSelectQuery; class ASTFunction; class MergeTreeData; @@ -35,7 +33,10 @@ class MergeTreeWhereOptimizer : private boost::noncopyable { public: MergeTreeWhereOptimizer( - ASTPtr & query, const Context & context, const MergeTreeData & data, const Names & column_names, + SelectQueryInfo & query_info, + const Context & context, + const MergeTreeData & data, + const Names & column_names, Poco::Logger * log); private: @@ -76,6 +77,7 @@ private: const string_set_t primary_key_columns; const string_set_t table_columns; const Block block_with_constants; + const PreparedSets & prepared_sets; Poco::Logger * log; std::unordered_map column_sizes{}; std::size_t total_column_size{}; diff --git a/dbms/src/Storages/MergeTree/PKCondition.cpp b/dbms/src/Storages/MergeTree/PKCondition.cpp index 0ca77630a6f..edcd7b024d8 100644 --- a/dbms/src/Storages/MergeTree/PKCondition.cpp +++ b/dbms/src/Storages/MergeTree/PKCondition.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -219,9 +218,13 @@ Block PKCondition::getBlockWithConstants( } -PKCondition::PKCondition(const ASTPtr & query, const Context & context, const NamesAndTypesList & all_columns, - const SortDescription & sort_descr_, ExpressionActionsPtr pk_expr_) - : sort_descr(sort_descr_), pk_expr(pk_expr_) +PKCondition::PKCondition( + const SelectQueryInfo & query_info, + const Context & context, + const NamesAndTypesList & all_columns, + const SortDescription & sort_descr_, + ExpressionActionsPtr pk_expr_) + : sort_descr(sort_descr_), pk_expr(pk_expr_), prepared_sets(query_info.sets) { for (size_t i = 0; i < sort_descr.size(); ++i) { @@ -233,10 +236,10 @@ PKCondition::PKCondition(const ASTPtr & query, const Context & context, const Na /** Evaluation of expressions that depend only on constants. * For the index to be used, if it is written, for example `WHERE Date = toDate(now())`. */ - Block block_with_constants = getBlockWithConstants(query, context, all_columns); + Block block_with_constants = getBlockWithConstants(query_info.query, context, all_columns); /// Trasform WHERE section to Reverse Polish notation - const ASTSelectQuery & select = typeid_cast(*query); + const ASTSelectQuery & select = typeid_cast(*query_info.query); if (select.where_expression) { traverseAST(select.where_expression, context, block_with_constants); @@ -545,7 +548,7 @@ bool PKCondition::atomFromAST(const ASTPtr & node, const Context & context, Bloc key_arg_pos = 1; is_constant_transformed = true; } - else if (typeid_cast(args[1].get()) + else if (prepared_sets.count(args[1].get()) && isPrimaryKeyPossiblyWrappedByMonotonicFunctions(args[0], context, key_column_num, key_expr_type, chain)) { key_arg_pos = 0; @@ -899,10 +902,10 @@ bool PKCondition::mayBeTrueInRangeImpl(const std::vector & key_ranges, co { auto in_func = typeid_cast(element.in_function.get()); const ASTs & args = typeid_cast(*in_func->arguments).children; - auto ast_set = typeid_cast(args[1].get()); - if (in_func && ast_set) + PreparedSets::const_iterator it = prepared_sets.find(args[1].get()); + if (in_func && it != prepared_sets.end()) { - rpn_stack.push_back(ast_set->set->mayBeTrueInRange(*key_range)); + rpn_stack.push_back(it->second->mayBeTrueInRange(*key_range)); if (element.function == RPNElement::FUNCTION_NOT_IN_SET) rpn_stack.back() = !rpn_stack.back(); } @@ -962,14 +965,6 @@ bool PKCondition::mayBeTrueAfter( } -static const ASTSet & inFunctionToSet(const ASTPtr & in_function) -{ - const auto & in_func = typeid_cast(*in_function); - const auto & args = typeid_cast(*in_func.arguments).children; - const auto & ast_set = typeid_cast(*args[1]); - return ast_set; -} - String PKCondition::RPNElement::toString() const { auto print_wrapped_column = [this](std::ostringstream & ss) @@ -999,7 +994,7 @@ String PKCondition::RPNElement::toString() const { ss << "("; print_wrapped_column(ss); - ss << (function == FUNCTION_IN_SET ? " in " : " notIn ") << inFunctionToSet(in_function).set->describe(); + ss << (function == FUNCTION_IN_SET ? " in set" : " notIn set"); ss << ")"; return ss.str(); } diff --git a/dbms/src/Storages/MergeTree/PKCondition.h b/dbms/src/Storages/MergeTree/PKCondition.h index 2c15d49037b..98c1d32cec0 100644 --- a/dbms/src/Storages/MergeTree/PKCondition.h +++ b/dbms/src/Storages/MergeTree/PKCondition.h @@ -4,11 +4,13 @@ #include #include +#include #include #include #include #include #include +#include namespace DB @@ -200,7 +202,11 @@ class PKCondition { public: /// Does not include the SAMPLE section. all_columns - the set of all columns of the table. - PKCondition(const ASTPtr & query, const Context & context, const NamesAndTypesList & all_columns, const SortDescription & sort_descr, + PKCondition( + const SelectQueryInfo & query_info, + const Context & context, + const NamesAndTypesList & all_columns, + const SortDescription & sort_descr, ExpressionActionsPtr pk_expr); /// Whether the condition is feasible in the key range. @@ -324,6 +330,7 @@ private: SortDescription sort_descr; ColumnIndices pk_columns; ExpressionActionsPtr pk_expr; + PreparedSets prepared_sets; }; } diff --git a/dbms/src/Storages/SelectQueryInfo.h b/dbms/src/Storages/SelectQueryInfo.h new file mode 100644 index 00000000000..c74fd0dad79 --- /dev/null +++ b/dbms/src/Storages/SelectQueryInfo.h @@ -0,0 +1,32 @@ +#pragma once + +#include + + +namespace DB +{ + +class IAST; +using ASTPtr = std::shared_ptr; + +class Set; +using SetPtr = std::shared_ptr; + +/// Information about calculated sets in right hand side of IN. +using PreparedSets = std::unordered_map; + + +/** Query along with some additional data, + * that can be used during query processing + * inside storage engines. + */ +struct SelectQueryInfo +{ + ASTPtr query; + + /// Prepared sets are used for indices by storage engine. + /// Example: x IN (1, 2, 3) + PreparedSets sets; +}; + +} diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index b4c5d0d4468..9298bca5b12 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -114,7 +114,7 @@ private: BlockInputStreams StorageBuffer::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, @@ -131,7 +131,7 @@ BlockInputStreams StorageBuffer::read( if (destination.get() == this) throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP); - streams_from_dst = destination->read(column_names, query, context, processed_stage, max_block_size, num_streams); + streams_from_dst = destination->read(column_names, query_info, context, processed_stage, max_block_size, num_streams); } BlockInputStreams streams_from_buffers; @@ -144,7 +144,7 @@ BlockInputStreams StorageBuffer::read( */ if (processed_stage > QueryProcessingStage::FetchColumns) for (auto & stream : streams_from_buffers) - stream = InterpreterSelectQuery(query, context, processed_stage, 0, stream).execute().in; + stream = InterpreterSelectQuery(query_info.query, context, processed_stage, 0, stream).execute().in; streams_from_dst.insert(streams_from_dst.end(), streams_from_buffers.begin(), streams_from_buffers.end()); return streams_from_dst; diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h index 1fa84e1852d..fc2362c0aeb 100644 --- a/dbms/src/Storages/StorageBuffer.h +++ b/dbms/src/Storages/StorageBuffer.h @@ -58,7 +58,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageCloud.h b/dbms/src/Storages/StorageCloud.h index fcd72c46bb5..c1b2865d490 100644 --- a/dbms/src/Storages/StorageCloud.h +++ b/dbms/src/Storages/StorageCloud.h @@ -35,7 +35,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp index d3ec563d0c2..835b6f19f6e 100644 --- a/dbms/src/Storages/StorageDictionary.cpp +++ b/dbms/src/Storages/StorageDictionary.cpp @@ -68,7 +68,7 @@ StorageDictionary::StorageDictionary( BlockInputStreams StorageDictionary::read( const Names & column_names, - const ASTPtr& query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/StorageDictionary.h b/dbms/src/Storages/StorageDictionary.h index 8fbed455f74..8394b457169 100644 --- a/dbms/src/Storages/StorageDictionary.h +++ b/dbms/src/Storages/StorageDictionary.h @@ -41,7 +41,7 @@ public: std::string getTableName() const override { return table_name; } const NamesAndTypesList & getColumnsListImpl() const override { return *columns; } BlockInputStreams read(const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size = DEFAULT_BLOCK_SIZE, diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 7b1b457b7ba..02684f706a0 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -189,7 +189,7 @@ StoragePtr StorageDistributed::createWithOwnCluster( BlockInputStreams StorageDistributed::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, @@ -206,7 +206,7 @@ BlockInputStreams StorageDistributed::read( : QueryProcessingStage::WithMergeableState; const auto & modified_query_ast = rewriteSelectQuery( - query, remote_database, remote_table); + query_info.query, remote_database, remote_table); Tables external_tables; @@ -218,7 +218,6 @@ BlockInputStreams StorageDistributed::read( /** The functionality of shard_multiplexing is not completed - turn it off. * (Because connecting to different shards within a single thread is not done in parallel.) - * For more information, see https: //███████████.yandex-team.ru/METR-18300 */ //bool enable_shard_multiplexing = !(ast.order_expression_list && !ast.group_expression_list); bool enable_shard_multiplexing = false; @@ -354,7 +353,6 @@ void StorageDistributed::reshardPartitions( /** The functionality of shard_multiplexing is not completed - turn it off. * (Because connecting to different shards within a single thread is not done in parallel.) - * For more information, see https: //███████████.yandex-team.ru/METR-18300 */ bool enable_shard_multiplexing = false; @@ -433,7 +431,6 @@ BlockInputStreams StorageDistributed::describe(const Context & context, const Se /** The functionality of shard_multiplexing is not completed - turn it off. * (Because connecting connections to different shards within a single thread is not done in parallel.) - * For more information, see https://███████████.yandex-team.ru/METR-18300 */ bool enable_shard_multiplexing = false; diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index 4cd3cf19bd3..128f0da1efb 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -57,7 +57,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index cdf5a21998a..a9e4bf53a5a 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -155,7 +155,7 @@ private: BlockInputStreams StorageFile::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageFile.h b/dbms/src/Storages/StorageFile.h index d3e8d911462..4e04300963a 100644 --- a/dbms/src/Storages/StorageFile.h +++ b/dbms/src/Storages/StorageFile.h @@ -74,7 +74,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 7931a5e7e9d..2d002e0c1e2 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -775,7 +775,7 @@ const Marks & StorageLog::getMarksWithRealRowCount() const BlockInputStreams StorageLog::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageLog.h b/dbms/src/Storages/StorageLog.h index f15aa6cf860..c9996b169e4 100644 --- a/dbms/src/Storages/StorageLog.h +++ b/dbms/src/Storages/StorageLog.h @@ -52,7 +52,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 3f29b040e52..117733e41e4 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -112,13 +112,13 @@ bool StorageMaterializedView::hasColumn(const String & column_name) const BlockInputStreams StorageMaterializedView::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, const unsigned num_streams) { - return getInnerTable()->read(column_names, query, context, processed_stage, max_block_size, num_streams); + return getInnerTable()->read(column_names, query_info, context, processed_stage, max_block_size, num_streams); } BlockOutputStreamPtr StorageMaterializedView::write(const ASTPtr & query, const Settings & settings) diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h index 9d32cabf0de..3ef88924ebd 100644 --- a/dbms/src/Storages/StorageMaterializedView.h +++ b/dbms/src/Storages/StorageMaterializedView.h @@ -43,7 +43,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp index faa548f9e8f..abd45d365c9 100644 --- a/dbms/src/Storages/StorageMemory.cpp +++ b/dbms/src/Storages/StorageMemory.cpp @@ -96,7 +96,7 @@ StorageMemory::StorageMemory( BlockInputStreams StorageMemory::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageMemory.h b/dbms/src/Storages/StorageMemory.h index 31e3e2176cd..32e476e3973 100644 --- a/dbms/src/Storages/StorageMemory.h +++ b/dbms/src/Storages/StorageMemory.h @@ -36,7 +36,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index dddcc46e54d..62da7886841 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -109,7 +109,7 @@ static Names collectIdentifiersInFirstLevelOfSelectQuery(ASTPtr ast) BlockInputStreams StorageMerge::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, @@ -131,6 +131,8 @@ BlockInputStreams StorageMerge::read( */ StorageListWithLocks selected_tables = getSelectedTables(); + const ASTPtr & query = query_info.query; + /// If PREWHERE is used in query, you need to make sure that all tables support this. if (typeid_cast(*query).prewhere_expression) for (const auto & elem : selected_tables) @@ -178,7 +180,7 @@ BlockInputStreams StorageMerge::read( QueryProcessingStage::Enum processed_stage_in_source_table = processed_stage; source_streams = table->read( real_column_names, - modified_query_ast, + { modified_query_ast, query_info.sets }, modified_context, processed_stage_in_source_table, max_block_size, @@ -206,7 +208,7 @@ BlockInputStreams StorageMerge::read( QueryProcessingStage::Enum processed_stage_in_source_table = processed_stage; BlockInputStreams streams = table->read( real_column_names, - modified_query_ast, + { modified_query_ast, query_info.sets }, modified_context, processed_stage_in_source_table, max_block_size, diff --git a/dbms/src/Storages/StorageMerge.h b/dbms/src/Storages/StorageMerge.h index 05827090ed1..835f60a586e 100644 --- a/dbms/src/Storages/StorageMerge.h +++ b/dbms/src/Storages/StorageMerge.h @@ -34,7 +34,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 29611dfdeb1..c9962f894a2 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -101,13 +101,13 @@ StorageMergeTree::~StorageMergeTree() BlockInputStreams StorageMergeTree::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, const unsigned num_streams) { - return reader.read(column_names, query, context, processed_stage, max_block_size, num_streams, nullptr, 0); + return reader.read(column_names, query_info, context, processed_stage, max_block_size, num_streams, nullptr, 0); } BlockOutputStreamPtr StorageMergeTree::write(const ASTPtr & query, const Settings & settings) diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index fc70547b981..bb09cf38e8c 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -51,7 +51,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageNull.h b/dbms/src/Storages/StorageNull.h index 7717cfc07d6..fba3e8bd8eb 100644 --- a/dbms/src/Storages/StorageNull.h +++ b/dbms/src/Storages/StorageNull.h @@ -26,7 +26,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index d1a71889b44..62421021cce 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -2288,7 +2288,7 @@ StorageReplicatedMergeTree::~StorageReplicatedMergeTree() BlockInputStreams StorageReplicatedMergeTree::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, @@ -2340,7 +2340,7 @@ BlockInputStreams StorageReplicatedMergeTree::read( } return reader.read( - column_names, query, context, processed_stage, max_block_size, num_streams, &part_index, max_block_number_to_read); + column_names, query_info, context, processed_stage, max_block_size, num_streams, &part_index, max_block_number_to_read); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index de19d356ae7..9f60fc077bf 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -123,7 +123,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageStripeLog.cpp b/dbms/src/Storages/StorageStripeLog.cpp index daf9b930f8d..fd1da079b2d 100644 --- a/dbms/src/Storages/StorageStripeLog.cpp +++ b/dbms/src/Storages/StorageStripeLog.cpp @@ -4,9 +4,6 @@ #include #include -#include -#include - #include #include @@ -18,9 +15,6 @@ #include #include -#include -#include - #include #include #include @@ -32,7 +26,6 @@ #include #include -#include namespace DB @@ -222,7 +215,7 @@ void StorageStripeLog::rename(const String & new_path_to_db, const String & new_ BlockInputStreams StorageStripeLog::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/StorageStripeLog.h b/dbms/src/Storages/StorageStripeLog.h index 896cc08eaa9..913277ade23 100644 --- a/dbms/src/Storages/StorageStripeLog.h +++ b/dbms/src/Storages/StorageStripeLog.h @@ -31,7 +31,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 89bae8d0d5c..9b86ed2df54 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -527,7 +527,7 @@ void StorageTinyLog::rename(const String & new_path_to_db, const String & new_da BlockInputStreams StorageTinyLog::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/StorageTinyLog.h b/dbms/src/Storages/StorageTinyLog.h index f9ea9392764..87c34c3c7fd 100644 --- a/dbms/src/Storages/StorageTinyLog.h +++ b/dbms/src/Storages/StorageTinyLog.h @@ -31,7 +31,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageTrivialBuffer.cpp b/dbms/src/Storages/StorageTrivialBuffer.cpp index 0c143a7d0a3..ee403aa62d6 100644 --- a/dbms/src/Storages/StorageTrivialBuffer.cpp +++ b/dbms/src/Storages/StorageTrivialBuffer.cpp @@ -111,7 +111,7 @@ private: BlockInputStreams StorageTrivialBuffer::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, @@ -130,8 +130,8 @@ BlockInputStreams StorageTrivialBuffer::read( throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP); - streams = destination->read(column_names, query, context, - processed_stage, max_block_size, num_streams); + streams = destination->read(column_names, query_info, context, + processed_stage, max_block_size, num_streams); } BlockInputStreams streams_from_buffers; @@ -156,12 +156,13 @@ BlockInputStreams StorageTrivialBuffer::read( */ if (processed_stage > QueryProcessingStage::FetchColumns) for (auto & stream : streams_from_buffers) - stream = InterpreterSelectQuery(query, context, processed_stage, 0, stream).execute().in; + stream = InterpreterSelectQuery(query_info.query, context, processed_stage, 0, stream).execute().in; streams.insert(streams.end(), streams_from_buffers.begin(), streams_from_buffers.end()); return streams; } + template void StorageTrivialBuffer::addBlock(const Block & block, DeduplicationController & deduplication_controller) { @@ -186,6 +187,7 @@ void StorageTrivialBuffer::addBlock(const Block & block, DeduplicationController } } + void StorageTrivialBuffer::flush(bool check_thresholds, bool is_called_from_background) { Block block_to_write; @@ -273,6 +275,7 @@ void StorageTrivialBuffer::flush(bool check_thresholds, bool is_called_from_back } + class TrivialBufferBlockOutputStream : public IBlockOutputStream { public: @@ -334,11 +337,13 @@ private: StorageTrivialBuffer & buffer; }; + BlockOutputStreamPtr StorageTrivialBuffer::write(const ASTPtr & query, const Settings & settings) { return std::make_shared(*this); } + void StorageTrivialBuffer::startup() { flush_thread = std::thread(&StorageTrivialBuffer::flushThread, this); @@ -363,6 +368,7 @@ void StorageTrivialBuffer::shutdown() } + /** NOTE If you do OPTIMIZE after insertion, * it does not guarantee that all data will be in destination table at the time of * next SELECT just after OPTIMIZE. @@ -393,7 +399,6 @@ bool StorageTrivialBuffer::optimize(const ASTPtr & query, const String & partiti } - bool StorageTrivialBuffer::checkThresholds( const time_t current_time, const size_t additional_rows, const size_t additional_bytes) const { @@ -408,6 +413,7 @@ bool StorageTrivialBuffer::checkThresholds( } + bool StorageTrivialBuffer::checkThresholdsImpl(const size_t rows, const size_t bytes, const time_t time_passed) const { @@ -438,6 +444,7 @@ bool StorageTrivialBuffer::checkThresholdsImpl(const size_t rows, const size_t b return false; } + void StorageTrivialBuffer::flushThread() { setThreadName("BufferFlush"); @@ -456,6 +463,7 @@ void StorageTrivialBuffer::flushThread() while (!shutdown_event.tryWait(1000)); } + void StorageTrivialBuffer::writeBlockToDestination(const Block & block, StoragePtr table) { if (no_destination || !block) @@ -518,6 +526,7 @@ void StorageTrivialBuffer::writeBlockToDestination(const Block & block, StorageP block_io.out->writeSuffix(); } + void StorageTrivialBuffer::alter( const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) diff --git a/dbms/src/Storages/StorageTrivialBuffer.h b/dbms/src/Storages/StorageTrivialBuffer.h index 3d3d7957463..a1131c1d304 100644 --- a/dbms/src/Storages/StorageTrivialBuffer.h +++ b/dbms/src/Storages/StorageTrivialBuffer.h @@ -52,7 +52,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 933ee8fae23..14e5dc93508 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -77,7 +77,7 @@ void StorageView::extractDependentTable(const ASTSelectQuery & query) BlockInputStreams StorageView::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/StorageView.h b/dbms/src/Storages/StorageView.h index ea68a3dcea6..cfbb207aaf2 100644 --- a/dbms/src/Storages/StorageView.h +++ b/dbms/src/Storages/StorageView.h @@ -27,7 +27,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp index 7da188ae980..0accd9ec4d2 100644 --- a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp +++ b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp @@ -26,7 +26,7 @@ StorageSystemAsynchronousMetrics::StorageSystemAsynchronousMetrics(const std::st BlockInputStreams StorageSystemAsynchronousMetrics::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.h b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.h index af42c360619..7224c158b9c 100644 --- a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.h +++ b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.h @@ -25,7 +25,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp index a41b4b2a2e8..634b541ed05 100644 --- a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp +++ b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp @@ -23,7 +23,7 @@ StorageSystemBuildOptions::StorageSystemBuildOptions(const std::string & name_) BlockInputStreams StorageSystemBuildOptions::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemBuildOptions.h b/dbms/src/Storages/System/StorageSystemBuildOptions.h index 13883b76791..5f212bb79a2 100644 --- a/dbms/src/Storages/System/StorageSystemBuildOptions.h +++ b/dbms/src/Storages/System/StorageSystemBuildOptions.h @@ -24,7 +24,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemClusters.cpp b/dbms/src/Storages/System/StorageSystemClusters.cpp index e97ff0da889..fb3b116030d 100644 --- a/dbms/src/Storages/System/StorageSystemClusters.cpp +++ b/dbms/src/Storages/System/StorageSystemClusters.cpp @@ -31,7 +31,7 @@ StorageSystemClusters::StorageSystemClusters(const std::string & name_) BlockInputStreams StorageSystemClusters::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemClusters.h b/dbms/src/Storages/System/StorageSystemClusters.h index 029da2f6dbf..24c28221392 100644 --- a/dbms/src/Storages/System/StorageSystemClusters.h +++ b/dbms/src/Storages/System/StorageSystemClusters.h @@ -26,7 +26,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp index eacbddd761a..a6dcf155227 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemColumns.cpp @@ -34,7 +34,7 @@ StorageSystemColumns::StorageSystemColumns(const std::string & name_) BlockInputStreams StorageSystemColumns::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, @@ -57,7 +57,7 @@ BlockInputStreams StorageSystemColumns::read( block.insert(ColumnWithTypeAndName(database_column, std::make_shared(), "database")); /// Filter block with `database` column. - VirtualColumnUtils::filterBlockWithQuery(query, block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, block, context); if (!block.rows()) return BlockInputStreams(); @@ -95,7 +95,7 @@ BlockInputStreams StorageSystemColumns::read( } /// Filter block with `database` and `table` columns. - VirtualColumnUtils::filterBlockWithQuery(query, block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, block, context); if (!block.rows()) return BlockInputStreams(); diff --git a/dbms/src/Storages/System/StorageSystemColumns.h b/dbms/src/Storages/System/StorageSystemColumns.h index dc7f5c23726..3e00f380375 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.h +++ b/dbms/src/Storages/System/StorageSystemColumns.h @@ -21,7 +21,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemDatabases.cpp b/dbms/src/Storages/System/StorageSystemDatabases.cpp index fd2c8f0a2e1..d49db3b3093 100644 --- a/dbms/src/Storages/System/StorageSystemDatabases.cpp +++ b/dbms/src/Storages/System/StorageSystemDatabases.cpp @@ -23,7 +23,7 @@ StorageSystemDatabases::StorageSystemDatabases(const std::string & name_) BlockInputStreams StorageSystemDatabases::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemDatabases.h b/dbms/src/Storages/System/StorageSystemDatabases.h index be233aa4838..a93b107b132 100644 --- a/dbms/src/Storages/System/StorageSystemDatabases.h +++ b/dbms/src/Storages/System/StorageSystemDatabases.h @@ -23,7 +23,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemDictionaries.cpp b/dbms/src/Storages/System/StorageSystemDictionaries.cpp index 280728d21f0..4c5c7544ea4 100644 --- a/dbms/src/Storages/System/StorageSystemDictionaries.cpp +++ b/dbms/src/Storages/System/StorageSystemDictionaries.cpp @@ -42,7 +42,7 @@ StorageSystemDictionaries::StorageSystemDictionaries(const std::string & name) BlockInputStreams StorageSystemDictionaries::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemDictionaries.h b/dbms/src/Storages/System/StorageSystemDictionaries.h index 87063398556..3838681ee45 100644 --- a/dbms/src/Storages/System/StorageSystemDictionaries.h +++ b/dbms/src/Storages/System/StorageSystemDictionaries.h @@ -19,7 +19,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemEvents.cpp b/dbms/src/Storages/System/StorageSystemEvents.cpp index bd23cde46ee..72926206d04 100644 --- a/dbms/src/Storages/System/StorageSystemEvents.cpp +++ b/dbms/src/Storages/System/StorageSystemEvents.cpp @@ -23,7 +23,7 @@ StorageSystemEvents::StorageSystemEvents(const std::string & name_) BlockInputStreams StorageSystemEvents::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemEvents.h b/dbms/src/Storages/System/StorageSystemEvents.h index b9473b0f1d0..05ab28b83af 100644 --- a/dbms/src/Storages/System/StorageSystemEvents.h +++ b/dbms/src/Storages/System/StorageSystemEvents.h @@ -23,7 +23,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemFunctions.cpp b/dbms/src/Storages/System/StorageSystemFunctions.cpp index 97820603420..1c815fa74d1 100644 --- a/dbms/src/Storages/System/StorageSystemFunctions.cpp +++ b/dbms/src/Storages/System/StorageSystemFunctions.cpp @@ -25,7 +25,7 @@ StorageSystemFunctions::StorageSystemFunctions(const std::string & name_) BlockInputStreams StorageSystemFunctions::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemFunctions.h b/dbms/src/Storages/System/StorageSystemFunctions.h index 14c1eec60c4..ae4769eb1f9 100644 --- a/dbms/src/Storages/System/StorageSystemFunctions.h +++ b/dbms/src/Storages/System/StorageSystemFunctions.h @@ -23,7 +23,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemGraphite.cpp b/dbms/src/Storages/System/StorageSystemGraphite.cpp index 0a9f8052f00..016aeaa304d 100644 --- a/dbms/src/Storages/System/StorageSystemGraphite.cpp +++ b/dbms/src/Storages/System/StorageSystemGraphite.cpp @@ -127,7 +127,7 @@ StorageSystemGraphite::StorageSystemGraphite(const std::string & name_) BlockInputStreams StorageSystemGraphite::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemGraphite.h b/dbms/src/Storages/System/StorageSystemGraphite.h index 7b1668d1e3a..b587a6a823c 100644 --- a/dbms/src/Storages/System/StorageSystemGraphite.h +++ b/dbms/src/Storages/System/StorageSystemGraphite.h @@ -17,7 +17,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemMerges.cpp b/dbms/src/Storages/System/StorageSystemMerges.cpp index 9cd80cb682a..81f20aab9e6 100644 --- a/dbms/src/Storages/System/StorageSystemMerges.cpp +++ b/dbms/src/Storages/System/StorageSystemMerges.cpp @@ -37,7 +37,7 @@ StorageSystemMerges::StorageSystemMerges(const std::string & name) BlockInputStreams StorageSystemMerges::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemMerges.h b/dbms/src/Storages/System/StorageSystemMerges.h index 28ebaca5686..3c8b1b01019 100644 --- a/dbms/src/Storages/System/StorageSystemMerges.h +++ b/dbms/src/Storages/System/StorageSystemMerges.h @@ -20,7 +20,7 @@ public: const NamesAndTypesList & getColumnsListImpl() const override { return columns; } BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemMetrics.cpp b/dbms/src/Storages/System/StorageSystemMetrics.cpp index 91527f71194..28071ba5bf3 100644 --- a/dbms/src/Storages/System/StorageSystemMetrics.cpp +++ b/dbms/src/Storages/System/StorageSystemMetrics.cpp @@ -24,7 +24,7 @@ StorageSystemMetrics::StorageSystemMetrics(const std::string & name_) BlockInputStreams StorageSystemMetrics::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemMetrics.h b/dbms/src/Storages/System/StorageSystemMetrics.h index 286bd9071b8..c19f7ef18ae 100644 --- a/dbms/src/Storages/System/StorageSystemMetrics.h +++ b/dbms/src/Storages/System/StorageSystemMetrics.h @@ -23,7 +23,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp index d1c8bc2c6fb..8bdaa3d3c37 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.cpp +++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp @@ -57,7 +57,7 @@ StorageSystemNumbers::StorageSystemNumbers(const std::string & name_, bool multi BlockInputStreams StorageSystemNumbers::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemNumbers.h b/dbms/src/Storages/System/StorageSystemNumbers.h index 0d9673ea1a4..d151af9cddf 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.h +++ b/dbms/src/Storages/System/StorageSystemNumbers.h @@ -30,7 +30,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemOne.cpp b/dbms/src/Storages/System/StorageSystemOne.cpp index cd0e5fcda84..b8356aa23e5 100644 --- a/dbms/src/Storages/System/StorageSystemOne.cpp +++ b/dbms/src/Storages/System/StorageSystemOne.cpp @@ -18,7 +18,7 @@ StorageSystemOne::StorageSystemOne(const std::string & name_) BlockInputStreams StorageSystemOne::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemOne.h b/dbms/src/Storages/System/StorageSystemOne.h index 7f714e5764d..6134789cbed 100644 --- a/dbms/src/Storages/System/StorageSystemOne.h +++ b/dbms/src/Storages/System/StorageSystemOne.h @@ -26,7 +26,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemParts.cpp b/dbms/src/Storages/System/StorageSystemParts.cpp index e56f60c7380..70fd42f14f8 100644 --- a/dbms/src/Storages/System/StorageSystemParts.cpp +++ b/dbms/src/Storages/System/StorageSystemParts.cpp @@ -47,7 +47,7 @@ StorageSystemParts::StorageSystemParts(const std::string & name_) BlockInputStreams StorageSystemParts::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, @@ -73,7 +73,7 @@ BlockInputStreams StorageSystemParts::read( block_to_filter.insert(ColumnWithTypeAndName(database_column, std::make_shared(), "database")); /// Filter block_to_filter with column 'database'. - VirtualColumnUtils::filterBlockWithQuery(query, block_to_filter, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, block_to_filter, context); if (!block_to_filter.rows()) return BlockInputStreams(); @@ -129,7 +129,7 @@ BlockInputStreams StorageSystemParts::read( } /// Filter block_to_filter with columns 'database', 'table', 'engine', 'active'. - VirtualColumnUtils::filterBlockWithQuery(query, block_to_filter, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, block_to_filter, context); /// If all was filtered out. if (!block_to_filter.rows()) diff --git a/dbms/src/Storages/System/StorageSystemParts.h b/dbms/src/Storages/System/StorageSystemParts.h index 22bcfab456a..17c6a7f4e5c 100644 --- a/dbms/src/Storages/System/StorageSystemParts.h +++ b/dbms/src/Storages/System/StorageSystemParts.h @@ -23,7 +23,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemProcesses.cpp b/dbms/src/Storages/System/StorageSystemProcesses.cpp index 4a43d9adcf1..4b5d3e4fa3e 100644 --- a/dbms/src/Storages/System/StorageSystemProcesses.cpp +++ b/dbms/src/Storages/System/StorageSystemProcesses.cpp @@ -55,7 +55,7 @@ StorageSystemProcesses::StorageSystemProcesses(const std::string & name_) BlockInputStreams StorageSystemProcesses::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemProcesses.h b/dbms/src/Storages/System/StorageSystemProcesses.h index f3ca99590c0..c86aacdf990 100644 --- a/dbms/src/Storages/System/StorageSystemProcesses.h +++ b/dbms/src/Storages/System/StorageSystemProcesses.h @@ -23,7 +23,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemReplicas.cpp b/dbms/src/Storages/System/StorageSystemReplicas.cpp index af85306d14b..9a1074154ae 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicas.cpp @@ -50,7 +50,7 @@ StorageSystemReplicas::StorageSystemReplicas(const std::string & name_) BlockInputStreams StorageSystemReplicas::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, @@ -98,14 +98,14 @@ BlockInputStreams StorageSystemReplicas::read( { Block filtered_block { col_database, col_table, col_engine }; - VirtualColumnUtils::filterBlockWithQuery(query, filtered_block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context); if (!filtered_block.rows()) return BlockInputStreams(); - col_database = filtered_block.getByName("database"); - col_table = filtered_block.getByName("table"); - col_engine = filtered_block.getByName("engine"); + col_database = filtered_block.getByName("database"); + col_table = filtered_block.getByName("table"); + col_engine = filtered_block.getByName("engine"); } ColumnWithTypeAndName col_is_leader{std::make_shared(), std::make_shared(), "is_leader"}; diff --git a/dbms/src/Storages/System/StorageSystemReplicas.h b/dbms/src/Storages/System/StorageSystemReplicas.h index 365556b0ad5..437cdcbeea8 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.h +++ b/dbms/src/Storages/System/StorageSystemReplicas.h @@ -23,7 +23,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp index 04d58c921d1..5c3c7f5d931 100644 --- a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -49,7 +49,7 @@ StorageSystemReplicationQueue::StorageSystemReplicationQueue(const std::string & BlockInputStreams StorageSystemReplicationQueue::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, @@ -80,7 +80,7 @@ BlockInputStreams StorageSystemReplicationQueue::read( { Block filtered_block { col_database_to_filter, col_table_to_filter }; - VirtualColumnUtils::filterBlockWithQuery(query, filtered_block, context); + VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context); if (!filtered_block.rows()) return BlockInputStreams(); diff --git a/dbms/src/Storages/System/StorageSystemReplicationQueue.h b/dbms/src/Storages/System/StorageSystemReplicationQueue.h index c884d2b637b..82f621f13eb 100644 --- a/dbms/src/Storages/System/StorageSystemReplicationQueue.h +++ b/dbms/src/Storages/System/StorageSystemReplicationQueue.h @@ -23,7 +23,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemSettings.cpp b/dbms/src/Storages/System/StorageSystemSettings.cpp index 8e3c1c630cc..59cb22d0cd8 100644 --- a/dbms/src/Storages/System/StorageSystemSettings.cpp +++ b/dbms/src/Storages/System/StorageSystemSettings.cpp @@ -24,7 +24,7 @@ StorageSystemSettings::StorageSystemSettings(const std::string & name_) BlockInputStreams StorageSystemSettings::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemSettings.h b/dbms/src/Storages/System/StorageSystemSettings.h index eafc0486053..e81e80b2814 100644 --- a/dbms/src/Storages/System/StorageSystemSettings.h +++ b/dbms/src/Storages/System/StorageSystemSettings.h @@ -23,7 +23,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index cce9d5641af..3f91703c507 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -52,7 +52,7 @@ static ColumnWithTypeAndName getFilteredDatabases(const ASTPtr & query, const Co BlockInputStreams StorageSystemTables::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, @@ -87,7 +87,7 @@ BlockInputStreams StorageSystemTables::read( col_meta_mod_time.column = std::make_shared(); block.insert(col_meta_mod_time); - ColumnWithTypeAndName filtered_databases_column = getFilteredDatabases(query, context); + ColumnWithTypeAndName filtered_databases_column = getFilteredDatabases(query_info.query, context); for (size_t row_number = 0; row_number < filtered_databases_column.column->size(); ++row_number) { diff --git a/dbms/src/Storages/System/StorageSystemTables.h b/dbms/src/Storages/System/StorageSystemTables.h index 61e2eda493c..324e5da8454 100644 --- a/dbms/src/Storages/System/StorageSystemTables.h +++ b/dbms/src/Storages/System/StorageSystemTables.h @@ -23,7 +23,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp index 0af2b6f7692..008f4bf639a 100644 --- a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp @@ -105,7 +105,7 @@ static String extractPath(const ASTPtr & query) BlockInputStreams StorageSystemZooKeeper::read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, @@ -114,7 +114,7 @@ BlockInputStreams StorageSystemZooKeeper::read( check(column_names); processed_stage = QueryProcessingStage::FetchColumns; - String path = extractPath(query); + String path = extractPath(query_info.query); if (path.empty()) throw Exception("SELECT from system.zookeeper table must contain condition like path = 'path' in WHERE clause."); diff --git a/dbms/src/Storages/System/StorageSystemZooKeeper.h b/dbms/src/Storages/System/StorageSystemZooKeeper.h index b9062521f86..445f04f51f7 100644 --- a/dbms/src/Storages/System/StorageSystemZooKeeper.h +++ b/dbms/src/Storages/System/StorageSystemZooKeeper.h @@ -23,7 +23,7 @@ public: BlockInputStreams read( const Names & column_names, - const ASTPtr & query, + const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size, diff --git a/dbms/src/Storages/tests/hit_log.cpp b/dbms/src/Storages/tests/hit_log.cpp index 47867a312d5..498a6f696c8 100644 --- a/dbms/src/Storages/tests/hit_log.cpp +++ b/dbms/src/Storages/tests/hit_log.cpp @@ -132,7 +132,7 @@ try QueryProcessingStage::Enum stage; - BlockInputStreamPtr in = table->read(column_names, 0, Context::createGlobal(), stage, 8192, 1)[0]; + BlockInputStreamPtr in = table->read(column_names, {}, Context::createGlobal(), stage, 8192, 1)[0]; RowOutputStreamPtr out_ = std::make_shared(out_buf, sample); BlockOutputStreamFromRowOutputStream out(out_); copyData(*in, out); diff --git a/dbms/src/Storages/tests/storage_log.cpp b/dbms/src/Storages/tests/storage_log.cpp index bf90c9a68f1..201ba39029e 100644 --- a/dbms/src/Storages/tests/storage_log.cpp +++ b/dbms/src/Storages/tests/storage_log.cpp @@ -9,6 +9,7 @@ #include #include #include +#include int main(int argc, char ** argv) @@ -68,7 +69,7 @@ try QueryProcessingStage::Enum stage; - BlockInputStreamPtr in = table->read(column_names, 0, Context::createGlobal(), stage, 8192, 1)[0]; + BlockInputStreamPtr in = table->read(column_names, {}, Context::createGlobal(), stage, 8192, 1)[0]; Block sample; { diff --git a/dbms/src/Storages/tests/system_numbers.cpp b/dbms/src/Storages/tests/system_numbers.cpp index f1baf1bd39b..bb660737bea 100644 --- a/dbms/src/Storages/tests/system_numbers.cpp +++ b/dbms/src/Storages/tests/system_numbers.cpp @@ -29,7 +29,7 @@ try QueryProcessingStage::Enum stage; - LimitBlockInputStream input(table->read(column_names, 0, Context::createGlobal(), stage, 10, 1)[0], 10, 96); + LimitBlockInputStream input(table->read(column_names, {}, Context::createGlobal(), stage, 10, 1)[0], 10, 96); RowOutputStreamPtr output_ = std::make_shared(out_buf, sample); BlockOutputStreamFromRowOutputStream output(output_); diff --git a/dbms/tests/queries/0_stateless/00372_cors_header.reference b/dbms/tests/queries/0_stateless/00372_cors_header.reference index 968ac3ef02d..e22493782f0 100644 --- a/dbms/tests/queries/0_stateless/00372_cors_header.reference +++ b/dbms/tests/queries/0_stateless/00372_cors_header.reference @@ -1,4 +1,3 @@ 1 0 0 -0 diff --git a/dbms/tests/queries/0_stateless/00372_cors_header.sh b/dbms/tests/queries/0_stateless/00372_cors_header.sh index 374b142c4fb..c9c4c7d38d7 100755 --- a/dbms/tests/queries/0_stateless/00372_cors_header.sh +++ b/dbms/tests/queries/0_stateless/00372_cors_header.sh @@ -2,5 +2,4 @@ curl -vsS 'http://localhost:8123/?add_http_cors_header=1' -H "Origin:smi2.ru" --data-binary @- <<< "SELECT 1" 2>&1 | grep -F "< Access-Control-Allow-Origin: *" | wc -l curl -vsS 'http://localhost:8123/?add_http_cors_header=0' -H "Origin:smi2.ru" --data-binary @- <<< "SELECT 1" 2>&1 | grep -F "< Access-Control-Allow-Origin: *" | wc -l -curl -vsS 'http://localhost:8123/' -H "Origin:smi2.ru" --data-binary @- <<< "SELECT 1" 2>&1 | grep -F "< Access-Control-Allow-Origin: *" | wc -l curl -vsS 'http://localhost:8123/?add_http_cors_header=1' --data-binary @- <<< "SELECT 1" 2>&1 | grep -F "< Access-Control-Allow-Origin: *" | wc -l diff --git a/dbms/tests/queries/0_stateless/00395_nullable.reference b/dbms/tests/queries/0_stateless/00395_nullable.reference index b426504a00d..bf3a34b2836 100644 --- a/dbms/tests/queries/0_stateless/00395_nullable.reference +++ b/dbms/tests/queries/0_stateless/00395_nullable.reference @@ -2,34 +2,31 @@ \N \N \N -1 1 [1] [1] a a ['a'] ['a'] [1] ['a'] -1 1 [1] [1] a a ['a'] ['a'] [1] [NULL] -1 1 [1] [1] a a ['a'] ['a'] [NULL] ['a'] -1 1 [1] [1] a a \N ['a'] [1] ['a'] -1 1 [1] [1] \N a ['a'] ['a'] [1] ['a'] -1 1 \N [1] a a ['a'] ['a'] [1] ['a'] -\N 1 [1] [1] a a ['a'] ['a'] [1] ['a'] -1 1 [1] [1] a a ['a'] ['a'] [1] ['a'] -1 1 [1] [1] a a ['a'] ['a'] [1] [NULL] -1 1 [1] [1] a a ['a'] ['a'] [NULL] ['a'] -1 1 [1] [1] a a \N ['a'] [1] ['a'] -1 1 [1] [1] \N a ['a'] ['a'] [1] ['a'] -1 1 \N [1] a a ['a'] ['a'] [1] ['a'] -\N 1 [1] [1] a a ['a'] ['a'] [1] ['a'] -1 1 [1] [1] a a ['a'] ['a'] [1] ['a'] -1 1 [1] [1] a a ['a'] ['a'] [1] [NULL] -1 1 [1] [1] a a ['a'] ['a'] [NULL] ['a'] -1 1 [1] [1] a a \N ['a'] [1] ['a'] -1 1 [1] [1] \N a ['a'] ['a'] [1] ['a'] -1 1 \N [1] a a ['a'] ['a'] [1] ['a'] -\N 1 [1] [1] a a ['a'] ['a'] [1] ['a'] -1 1 [1] [1] a a ['a'] ['a'] [1] ['a'] 0000-00-00 -1 1 [1] [1] a a ['a'] ['a'] [1] [NULL] 0000-00-00 -1 1 [1] [1] a a ['a'] ['a'] [NULL] ['a'] 0000-00-00 -1 1 [1] [1] a a \N ['a'] [1] ['a'] 0000-00-00 -1 1 [1] [1] \N a ['a'] ['a'] [1] ['a'] 0000-00-00 -1 1 \N [1] a a ['a'] ['a'] [1] ['a'] 0000-00-00 -\N 1 [1] [1] a a ['a'] ['a'] [1] ['a'] 0000-00-00 +1 1 a a [1] [1] ['a'] ['a'] 2000-01-01 +1 1 a a [1] [1] ['a'] [NULL] 2000-01-01 +1 1 a a [1] [NULL] ['a'] ['a'] 2000-01-01 +1 1 a \N [1] [1] ['a'] ['a'] 2000-01-01 +1 \N a a [1] [1] ['a'] ['a'] 2000-01-01 +1 1 a a [1] [1] ['a'] ['a'] 2000-01-01 +1 1 a a [1] [1] ['a'] [NULL] 2000-01-01 +1 1 a a [1] [NULL] ['a'] ['a'] 2000-01-01 +1 1 a \N [1] [1] ['a'] ['a'] 2000-01-01 +1 \N a a [1] [1] ['a'] ['a'] 2000-01-01 +1 1 a a [1] [1] ['a'] ['a'] 2000-01-01 +1 1 a a [1] [1] ['a'] [NULL] 2000-01-01 +1 1 a a [1] [NULL] ['a'] ['a'] 2000-01-01 +1 1 a \N [1] [1] ['a'] ['a'] 2000-01-01 +1 \N a a [1] [1] ['a'] ['a'] 2000-01-01 +1 1 a a [1] [1] ['a'] ['a'] 2000-01-01 +1 1 a a [1] [1] ['a'] [NULL] 2000-01-01 +1 1 a a [1] [NULL] ['a'] ['a'] 2000-01-01 +1 1 a \N [1] [1] ['a'] ['a'] 2000-01-01 +1 \N a a [1] [1] ['a'] ['a'] 2000-01-01 +1 1 a a [1] [1] ['a'] ['a'] 2000-01-01 +1 1 a a [1] [1] ['a'] [NULL] 2000-01-01 +1 1 a a [1] [NULL] ['a'] ['a'] 2000-01-01 +1 1 a \N [1] [1] ['a'] ['a'] 2000-01-01 +1 \N a a [1] [1] ['a'] ['a'] 2000-01-01 [2] 2 7 4 3 @@ -314,17 +311,4 @@ C 4 \N 3 \N 1 [0] 2 [1] 7 -\N 3 -[0] 2 -[1] 7 [NULL] 3 -[0] [0] 2 -[1] [1] 6 -[1] [NULL] 1 -\N [1] 1 -\N [NULL] 2 -[0] [0] -[1] [1] -[1] [NULL] -\N [1] -\N [NULL] diff --git a/dbms/tests/queries/0_stateless/00395_nullable.sql b/dbms/tests/queries/0_stateless/00395_nullable.sql index 21b65757234..34ba1b48ca8 100644 --- a/dbms/tests/queries/0_stateless/00395_nullable.sql +++ b/dbms/tests/queries/0_stateless/00395_nullable.sql @@ -6,87 +6,97 @@ SELECT 1 + NULL; SELECT abs(NULL); SELECT NULL + NULL; +/* MergeTree engine */ + +DROP TABLE IF EXISTS test.test1; +CREATE TABLE test.test1( +col1 UInt64, col2 Nullable(UInt64), +col3 String, col4 Nullable(String), +col5 Array(UInt64), col6 Array(Nullable(UInt64)), +col7 Array(String), col8 Array(Nullable(String)), +d Date) Engine = MergeTree(d, (col1, d), 8192); + +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, NULL, 'a', 'a', [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', NULL, [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [NULL], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [1], ['a'], [NULL], '2000-01-01'); +SELECT * FROM test.test1 ORDER BY col1,col2,col3,col4,col5,col6,col7,col8 ASC; + + /* Memory engine */ DROP TABLE IF EXISTS test.test1; CREATE TABLE test.test1( -col1 Nullable(UInt64), col2 UInt64, -col3 Nullable(Array(UInt64)), col4 Array(UInt64), -col5 Nullable(String), col6 String, -col7 Nullable(Array(String)), col8 Array(String), -col9 Array(Nullable(UInt64)), col10 Array(Nullable(String))) Engine = Memory; +col1 UInt64, col2 Nullable(UInt64), +col3 String, col4 Nullable(String), +col5 Array(UInt64), col6 Array(Nullable(UInt64)), +col7 Array(String), col8 Array(Nullable(String)), +d Date) Engine = Memory; -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], 'a', 'a', ['a'], ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (NULL, 1, [1], [1], 'a', 'a', ['a'], ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, NULL, [1], 'a', 'a', ['a'], ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], NULL, 'a', ['a'], ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], 'a', 'a', NULL, ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], 'a', 'a', ['a'], ['a'], [NULL], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], 'a', 'a', ['a'], ['a'], [1], [NULL]); -SELECT col1, col2, col3, col4, col5, col6, col7, col8, col9, col10 FROM test.test1 ORDER BY col1,col2,col3,col4,col5,col6,col7,col8,col9,col10 ASC; +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, NULL, 'a', 'a', [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', NULL, [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [NULL], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [1], ['a'], [NULL], '2000-01-01'); +SELECT * FROM test.test1 ORDER BY col1,col2,col3,col4,col5,col6,col7,col8 ASC; /* TinyLog engine */ DROP TABLE IF EXISTS test.test1; CREATE TABLE test.test1( -col1 Nullable(UInt64), col2 UInt64, -col3 Nullable(Array(UInt64)), col4 Array(UInt64), -col5 Nullable(String), col6 String, -col7 Nullable(Array(String)), col8 Array(String), -col9 Array(Nullable(UInt64)), col10 Array(Nullable(String))) Engine = TinyLog; +col1 UInt64, col2 Nullable(UInt64), +col3 String, col4 Nullable(String), +col5 Array(UInt64), col6 Array(Nullable(UInt64)), +col7 Array(String), col8 Array(Nullable(String)), +d Date) Engine = TinyLog; -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], 'a', 'a', ['a'], ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (NULL, 1, [1], [1], 'a', 'a', ['a'], ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, NULL, [1], 'a', 'a', ['a'], ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], NULL, 'a', ['a'], ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], 'a', 'a', NULL, ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], 'a', 'a', ['a'], ['a'], [NULL], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], 'a', 'a', ['a'], ['a'], [1], [NULL]); -SELECT col1, col2, col3, col4, col5, col6, col7, col8, col9, col10 FROM test.test1 ORDER BY col1,col2,col3,col4,col5,col6,col7,col8,col9,col10 ASC; +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, NULL, 'a', 'a', [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', NULL, [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [NULL], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [1], ['a'], [NULL], '2000-01-01'); +SELECT * FROM test.test1 ORDER BY col1,col2,col3,col4,col5,col6,col7,col8 ASC; /* Log engine */ DROP TABLE IF EXISTS test.test1; CREATE TABLE test.test1( -col1 Nullable(UInt64), col2 UInt64, -col3 Nullable(Array(UInt64)), col4 Array(UInt64), -col5 Nullable(String), col6 String, -col7 Nullable(Array(String)), col8 Array(String), -col9 Array(Nullable(UInt64)), col10 Array(Nullable(String))) Engine = Log; +col1 UInt64, col2 Nullable(UInt64), +col3 String, col4 Nullable(String), +col5 Array(UInt64), col6 Array(Nullable(UInt64)), +col7 Array(String), col8 Array(Nullable(String)), +d Date) Engine = Log; -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], 'a', 'a', ['a'], ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (NULL, 1, [1], [1], 'a', 'a', ['a'], ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, NULL, [1], 'a', 'a', ['a'], ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], NULL, 'a', ['a'], ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], 'a', 'a', NULL, ['a'], [1], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], 'a', 'a', ['a'], ['a'], [NULL], ['a']); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10) VALUES (1, 1, [1], [1], 'a', 'a', ['a'], ['a'], [1], [NULL]); -SELECT col1, col2, col3, col4, col5, col6, col7, col8, col9, col10 FROM test.test1 ORDER BY col1,col2,col3,col4,col5,col6,col7,col8,col9,col10 ASC; +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, NULL, 'a', 'a', [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', NULL, [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [NULL], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [1], ['a'], [NULL], '2000-01-01'); +SELECT * FROM test.test1 ORDER BY col1,col2,col3,col4,col5,col6,col7,col8 ASC; -/* MergeTree engine */ +/* StripeLog engine */ DROP TABLE IF EXISTS test.test1; CREATE TABLE test.test1( -col1 Nullable(UInt64), col2 UInt64, -col3 Nullable(Array(UInt64)), col4 Array(UInt64), -col5 Nullable(String), col6 String, -col7 Nullable(Array(String)), col8 Array(String), -col9 Array(Nullable(UInt64)), col10 Array(Nullable(String)), -col11 Date) Engine = MergeTree(col11, (col2, col11), 8192); +col1 UInt64, col2 Nullable(UInt64), +col3 String, col4 Nullable(String), +col5 Array(UInt64), col6 Array(Nullable(UInt64)), +col7 Array(String), col8 Array(Nullable(String)), +d Date) Engine = StripeLog; + +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, NULL, 'a', 'a', [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', NULL, [1], [1], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [NULL], ['a'], ['a'], '2000-01-01'); +INSERT INTO test.test1 VALUES (1, 1, 'a', 'a', [1], [1], ['a'], [NULL], '2000-01-01'); +SELECT * FROM test.test1 ORDER BY col1,col2,col3,col4,col5,col6,col7,col8 ASC; -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11) VALUES (1, 1, [1], [1], 'a', 'a', ['a'], ['a'], [1], ['a'], '1970-01-01'); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11) VALUES (NULL, 1, [1], [1], 'a', 'a', ['a'], ['a'], [1], ['a'], '1970-01-01'); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11) VALUES (1, 1, NULL, [1], 'a', 'a', ['a'], ['a'], [1], ['a'], '1970-01-01'); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11) VALUES (1, 1, [1], [1], NULL, 'a', ['a'], ['a'], [1], ['a'], '1970-01-01'); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11) VALUES (1, 1, [1], [1], 'a', 'a', NULL, ['a'], [1], ['a'], '1970-01-01'); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11) VALUES (1, 1, [1], [1], 'a', 'a', ['a'], ['a'], [NULL], ['a'], '1970-01-01'); -INSERT INTO test.test1(col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11) VALUES (1, 1, [1], [1], 'a', 'a', ['a'], ['a'], [1], [NULL], '1970-01-01'); -SELECT col1, col2, col3, col4, col5, col6, col7, col8, col9, col10, col11 FROM test.test1 ORDER BY col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11 ASC; /* Insert with expression */ DROP TABLE IF EXISTS test.test1; -CREATE TABLE test.test1(col1 Nullable(Array(UInt64))) Engine=Memory; +CREATE TABLE test.test1(col1 Array(Nullable(UInt64))) Engine=Memory; INSERT INTO test.test1(col1) VALUES ([1+1]); SELECT col1 FROM test.test1 ORDER BY col1 ASC; @@ -456,23 +466,6 @@ INSERT INTO test.test1(col1,col2,col3,col4) VALUES(NULL, 3, NULL, 'ACDEFBGH'); SELECT col1, col2, col3, count() FROM test.test1 GROUP BY col1, col2, col3 ORDER BY col1, col2, col3; -DROP TABLE IF EXISTS test.test1; -CREATE TABLE test.test1(col1 Nullable(Array(UInt8)), col2 String) ENGINE=TinyLog; -INSERT INTO test.test1(col1,col2) VALUES([0], 'ABCDEFGH'); -INSERT INTO test.test1(col1,col2) VALUES([0], 'BACDEFGH'); -INSERT INTO test.test1(col1,col2) VALUES([1], 'BCADEFGH'); -INSERT INTO test.test1(col1,col2) VALUES([1], 'BCDAEFGH'); -INSERT INTO test.test1(col1,col2) VALUES([1], 'BCDEAFGH'); -INSERT INTO test.test1(col1,col2) VALUES([1], 'BCDEFAGH'); -INSERT INTO test.test1(col1,col2) VALUES([1], 'BCDEFGAH'); -INSERT INTO test.test1(col1,col2) VALUES([1], 'BCDEFGHA'); -INSERT INTO test.test1(col1,col2) VALUES([1], 'ACBDEFGH'); -INSERT INTO test.test1(col1,col2) VALUES(NULL, 'ACDBEFGH'); -INSERT INTO test.test1(col1,col2) VALUES(NULL, 'ACDEBFGH'); -INSERT INTO test.test1(col1,col2) VALUES(NULL, 'ACDEFBGH'); - -SELECT col1, count() FROM test.test1 GROUP BY col1 ORDER BY col1; - DROP TABLE IF EXISTS test.test1; CREATE TABLE test.test1(col1 Array(Nullable(UInt8)), col2 String) ENGINE=TinyLog; INSERT INTO test.test1(col1,col2) VALUES([0], 'ABCDEFGH'); @@ -489,21 +482,3 @@ INSERT INTO test.test1(col1,col2) VALUES([NULL], 'ACDEBFGH'); INSERT INTO test.test1(col1,col2) VALUES([NULL], 'ACDEFBGH'); SELECT col1, count() FROM test.test1 GROUP BY col1 ORDER BY col1; - -DROP TABLE IF EXISTS test.test1; -CREATE TABLE test.test1(col1 Nullable(Array(UInt8)), col2 Array(Nullable(UInt8)), col3 String) ENGINE=TinyLog; -INSERT INTO test.test1(col1,col2,col3) VALUES([0], [0], 'ABCDEFGH'); -INSERT INTO test.test1(col1,col2,col3) VALUES([0], [0], 'BACDEFGH'); -INSERT INTO test.test1(col1,col2,col3) VALUES([1], [1], 'BCADEFGH'); -INSERT INTO test.test1(col1,col2,col3) VALUES([1], [1], 'BCDAEFGH'); -INSERT INTO test.test1(col1,col2,col3) VALUES([1], [1], 'BCDEAFGH'); -INSERT INTO test.test1(col1,col2,col3) VALUES([1], [1], 'BCDEFAGH'); -INSERT INTO test.test1(col1,col2,col3) VALUES([1], [1], 'BCDEFGAH'); -INSERT INTO test.test1(col1,col2,col3) VALUES([1], [1], 'BCDEFGHA'); -INSERT INTO test.test1(col1,col2,col3) VALUES([1], [NULL], 'ACBDEFGH'); -INSERT INTO test.test1(col1,col2,col3) VALUES(NULL, [1], 'ACDBEFGH'); -INSERT INTO test.test1(col1,col2,col3) VALUES(NULL, [NULL], 'ACDEBFGH'); -INSERT INTO test.test1(col1,col2,col3) VALUES(NULL, [NULL], 'ACDEFBGH'); - -SELECT col1, col2, count() FROM test.test1 GROUP BY col1, col2 ORDER BY col1, col2; -SELECT DISTINCT col1, col2 FROM test.test1 ORDER BY col1, col2; diff --git a/dbms/tests/queries/0_stateless/00446_clear_column_in_partition.reference b/dbms/tests/queries/0_stateless/00446_clear_column_in_partition.reference index cc523934d05..b219c65b7ff 100644 --- a/dbms/tests/queries/0_stateless/00446_clear_column_in_partition.reference +++ b/dbms/tests/queries/0_stateless/00446_clear_column_in_partition.reference @@ -3,18 +3,24 @@ 0 a 2 b all +2000-01-01 0 2000-01-01 1 a 2000-01-01 3 c +2000-02-01 0 2000-02-01 2 b 2000-02-01 4 d w/o i 1 +2000-01-01 0 2000-01-01 0 a 2000-01-01 0 c +2000-02-01 0 2000-02-01 2 b 2000-02-01 4 d w/o is 1 2000-01-01 0 2000-01-01 0 +2000-01-01 0 +2000-02-01 0 2000-02-01 2 b 2000-02-01 4 d w/o is 12 diff --git a/dbms/tests/queries/0_stateless/00446_clear_column_in_partition.sql b/dbms/tests/queries/0_stateless/00446_clear_column_in_partition.sql index 41c9ab1ad06..75935aa2491 100644 --- a/dbms/tests/queries/0_stateless/00446_clear_column_in_partition.sql +++ b/dbms/tests/queries/0_stateless/00446_clear_column_in_partition.sql @@ -13,8 +13,14 @@ DROP TABLE test.clear_column; DROP TABLE IF EXISTS test.clear_column1; DROP TABLE IF EXISTS test.clear_column2; -CREATE TABLE test.clear_column1 (d Date, i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/clear_column', '1', d, d, 8192); -CREATE TABLE test.clear_column2 (d Date, i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/clear_column', '2', d, d, 8192); +CREATE TABLE test.clear_column1 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/clear_column', '1', d, d, 8192); +CREATE TABLE test.clear_column2 (d Date, i Int64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/clear_column', '2', d, d, 8192); + +INSERT INTO test.clear_column1 (d) VALUES ('2000-01-01'), ('2000-02-01'); + +SET replication_alter_partitions_sync=2; +ALTER TABLE test.clear_column1 ADD COLUMN s String; +ALTER TABLE test.clear_column1 CLEAR COLUMN s IN PARTITION '200001'; INSERT INTO test.clear_column1 VALUES ('2000-01-01', 1, 'a'), ('2000-02-01', 2, 'b'); INSERT INTO test.clear_column1 VALUES ('2000-01-01', 3, 'c'), ('2000-02-01', 4, 'd'); @@ -22,8 +28,6 @@ INSERT INTO test.clear_column1 VALUES ('2000-01-01', 3, 'c'), ('2000-02-01', 4, SELECT 'all'; SELECT * FROM test.clear_column1 ORDER BY d, i, s; -SET replication_alter_partitions_sync=2; - SELECT 'w/o i 1'; ALTER TABLE test.clear_column1 CLEAR COLUMN i IN PARTITION '200001'; SELECT * FROM test.clear_column2 ORDER BY d, i, s; @@ -45,7 +49,7 @@ SELECT sum(data_uncompressed_bytes) FROM system.columns WHERE database='test' AN ALTER TABLE test.clear_column1 CLEAR COLUMN s IN PARTITION '200001'; ALTER TABLE test.clear_column1 CLEAR COLUMN s IN PARTITION '200002'; --- check optimize for non-leader replica +-- check optimize for non-leader replica (it is not related with CLEAR COLUMN) OPTIMIZE TABLE test.clear_column1; OPTIMIZE TABLE test.clear_column2; diff --git a/dbms/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent.sh b/dbms/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent.sh index 59921a7d75e..3ebb11cd33d 100755 --- a/dbms/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent.sh +++ b/dbms/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -e ch="clickhouse-client --stacktrace -q" @@ -8,6 +7,14 @@ $ch "DROP TABLE IF EXISTS test.clear_column2" $ch "CREATE TABLE test.clear_column1 (d Date, i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/clear_column', '1', d, d, 8192)" $ch "CREATE TABLE test.clear_column2 (d Date, i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/clear_column', '2', d, d, 8192)" +$ch "ALTER TABLE test.clear_column1 CLEAR COLUMN VasyaUnexistingColumn IN PARTITION '200001'" 1>/dev/null 2>/dev/null +rc=$? +if [ $rc -eq 0 ]; then + echo "An unexisisting column was ALTERed. Code: $rc" + exit -1 +fi + +set -e $ch "INSERT INTO test.clear_column1 VALUES ('2000-01-01', 1, 'a'), ('2000-02-01', 2, 'b')" $ch "INSERT INTO test.clear_column1 VALUES ('2000-01-01', 3, 'c'), ('2000-02-01', 4, 'd')" @@ -24,5 +31,5 @@ wait $ch "SELECT DISTINCT * FROM test.clear_column1 WHERE d != toDate('2000-03-01') ORDER BY d, i, s" $ch "SELECT DISTINCT * FROM test.clear_column2 WHERE d != toDate('2000-03-01') ORDER BY d, i, s" -$ch "DROP TABLE IF EXISTS test.clear_column1" -$ch "DROP TABLE IF EXISTS test.clear_column2" +#$ch "DROP TABLE IF EXISTS test.clear_column1" +#$ch "DROP TABLE IF EXISTS test.clear_column2" diff --git a/dbms/tests/queries/0_stateless/00477_parsing_data_types.reference b/dbms/tests/queries/0_stateless/00477_parsing_data_types.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00477_parsing_data_types.sql b/dbms/tests/queries/0_stateless/00477_parsing_data_types.sql new file mode 100644 index 00000000000..678bc5655dd --- /dev/null +++ b/dbms/tests/queries/0_stateless/00477_parsing_data_types.sql @@ -0,0 +1 @@ +CREATE TEMPORARY TABLE t (x Array( /* Hello */ UInt32 /* World */ )) ENGINE = Memory; diff --git a/libs/libdaemon/cmake/find_unwind.cmake b/libs/libdaemon/cmake/find_unwind.cmake index 591cf3d4d44..a3f3b1f0d2b 100644 --- a/libs/libdaemon/cmake/find_unwind.cmake +++ b/libs/libdaemon/cmake/find_unwind.cmake @@ -1,3 +1,6 @@ +include (CMakePushCheckState) +cmake_push_check_state () + if (CMAKE_SYSTEM MATCHES "Linux") option (USE_INTERNAL_UNWIND_LIBRARY "Set to FALSE to use system unwind library instead of bundled" ${NOT_UNBUNDLED}) else () @@ -41,3 +44,5 @@ elseif (CMAKE_SYSTEM MATCHES "Linux") endif () message (STATUS "Using unwind=${USE_UNWIND}: ${UNWIND_INCLUDE_DIR} : ${UNWIND_LIBRARY}") + +cmake_pop_check_state () diff --git a/release_lib.sh b/release_lib.sh index 56ef68bac10..abc392d9e1c 100644 --- a/release_lib.sh +++ b/release_lib.sh @@ -69,14 +69,14 @@ function gen_revision_author { fi - AUTHOR=$(git config --get user.name) + AUTHOR=$(git config --get user.name || echo ${USER}) export REVISION export AUTHOR } function get_revision_author { REVISION=$(get_revision) - AUTHOR=$(git config --get user.name) + AUTHOR=$(git config --get user.name || echo ${USER}) export REVISION export AUTHOR }