diff --git a/CMakeLists.txt b/CMakeLists.txt index 1727caea766..35c22526816 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -395,9 +395,10 @@ endif () # Turns on all external libs like s3, kafka, ODBC, ... option(ENABLE_LIBRARIES "Enable all external libraries by default" ON) -# We recommend avoiding this mode for production builds because we can't guarantee all needed libraries exist in your -# system. +# We recommend avoiding this mode for production builds because we can't guarantee +# all needed libraries exist in your system. # This mode exists for enthusiastic developers who are searching for trouble. +# The whole idea of using unknown version of libraries from the OS distribution is deeply flawed. # Useful for maintainers of OS packages. option (UNBUNDLED "Use system libraries instead of ones in contrib/" OFF) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 18072566d04..2435335f669 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -6,7 +6,7 @@ SET(VERSION_REVISION 54454) SET(VERSION_MAJOR 21) SET(VERSION_MINOR 9) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH f48c5af90c2ad51955d1ee3b6b05d006b03e4238) -SET(VERSION_DESCRIBE v21.9.1.1-prestable) -SET(VERSION_STRING 21.9.1.1) +SET(VERSION_GITHASH f063e44131a048ba2d9af8075f03700fd5ec3e69) +SET(VERSION_DESCRIBE v21.9.1.7770-prestable) +SET(VERSION_STRING 21.9.1.7770) # end of autochange diff --git a/contrib/libmetrohash/CMakeLists.txt b/contrib/libmetrohash/CMakeLists.txt index 9304cb3644c..4ec5a58717d 100644 --- a/contrib/libmetrohash/CMakeLists.txt +++ b/contrib/libmetrohash/CMakeLists.txt @@ -2,9 +2,5 @@ set (SRCS src/metrohash64.cpp src/metrohash128.cpp ) -if (HAVE_SSE42) # Not used. Pretty easy to port. - list (APPEND SRCS src/metrohash128crc.cpp) -endif () - add_library(metrohash ${SRCS}) target_include_directories(metrohash PUBLIC src) diff --git a/docker/packager/packager b/docker/packager/packager index c05c85d3e28..95b7fcd8568 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -151,8 +151,14 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ cmake_flags.append('-DENABLE_TESTS=1') cmake_flags.append('-DUSE_GTEST=1') + # "Unbundled" build is not suitable for any production usage. + # But it is occasionally used by some developers. + # The whole idea of using unknown version of libraries from the OS distribution is deeply flawed. + # We wish these developers good luck. if unbundled: - cmake_flags.append('-DUNBUNDLED=1 -DUSE_INTERNAL_RDKAFKA_LIBRARY=1 -DENABLE_ARROW=0 -DENABLE_AVRO=0 -DENABLE_ORC=0 -DENABLE_PARQUET=0') + # We also disable all CPU features except basic x86_64. + # It is only slightly related to "unbundled" build, but it is a good place to test if code compiles without these instruction sets. + cmake_flags.append('-DUNBUNDLED=1 -DUSE_INTERNAL_RDKAFKA_LIBRARY=1 -DENABLE_ARROW=0 -DENABLE_AVRO=0 -DENABLE_ORC=0 -DENABLE_PARQUET=0 -DENABLE_SSSE3=0 -DENABLE_SSE41=0 -DENABLE_SSE42=0 -DENABLE_PCLMULQDQ=0 -DENABLE_POPCNT=0 -DENABLE_AVX=0 -DENABLE_AVX2=0') if split_binary: cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') diff --git a/docker/test/stateless/process_functional_tests_result.py b/docker/test/stateless/process_functional_tests_result.py index e60424ad4d1..a42b0e68d88 100755 --- a/docker/test/stateless/process_functional_tests_result.py +++ b/docker/test/stateless/process_functional_tests_result.py @@ -105,6 +105,10 @@ def process_result(result_path): description += ", skipped: {}".format(skipped) if unknown != 0: description += ", unknown: {}".format(unknown) + + # Temporary green for tests with DatabaseReplicated: + if 1 == int(os.environ.get('USE_DATABASE_REPLICATED', 0)): + state = "success" else: state = "failure" description = "Output log doesn't exist" diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 0c900454cd0..1b1313e625c 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -79,7 +79,7 @@ For a description of parameters, see the [CREATE query description](../../../sql - `SAMPLE BY` — An expression for sampling. Optional. - If a sampling expression is used, the primary key must contain it. The result of sampling expression must be unsigned integer. Example: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. + If a sampling expression is used, the primary key must contain it. The result of a sampling expression must be an unsigned integer. Example: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. - `TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [between disks and volumes](#table_engine-mergetree-multiple-volumes). Optional. diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index db6eb8154ba..61ed34b686c 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -68,7 +68,7 @@ ORDER BY expr - `SAMPLE BY` — выражение для сэмплирования. Необязательный параметр. - Если используется выражение для сэмплирования, то первичный ключ должен содержать его. Пример: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. + Если используется выражение для сэмплирования, то первичный ключ должен содержать его. Результат выражения для сэмплирования должен быть беззнаковым целым числом. Пример: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. - `TTL` — список правил, определяющих длительности хранения строк, а также задающих правила перемещения частей на определённые тома или диски. Необязательный параметр. diff --git a/release b/release index 9484d79630a..de549595d43 100755 --- a/release +++ b/release @@ -71,9 +71,6 @@ then export DEB_CC=${DEB_CC=clang-10} export DEB_CXX=${DEB_CXX=clang++-10} EXTRAPACKAGES="$EXTRAPACKAGES clang-10 lld-10" -elif [[ $BUILD_TYPE == 'valgrind' ]]; then - MALLOC_OPTS="-DENABLE_TCMALLOC=0 -DENABLE_JEMALLOC=0" - VERSION_POSTFIX+="+valgrind" elif [[ $BUILD_TYPE == 'debug' ]]; then CMAKE_BUILD_TYPE=Debug VERSION_POSTFIX+="+debug" diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h index cc670fdf823..e5471b8a727 100644 --- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h +++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h @@ -6,11 +6,12 @@ #include #include #include +#include #include #include #include #include - +#include #include #if !defined(ARCADIA_BUILD) @@ -49,6 +50,8 @@ private: T value; public: + static constexpr bool is_nullable = false; + bool has() const { return has_value; @@ -469,6 +472,8 @@ private: char small_data[MAX_SMALL_STRING_SIZE]; /// Including the terminating zero. public: + static constexpr bool is_nullable = false; + bool has() const { return size >= 0; @@ -692,6 +697,8 @@ private: Field value; public: + static constexpr bool is_nullable = false; + bool has() const { return !value.isNull(); @@ -975,6 +982,68 @@ struct AggregateFunctionAnyLastData : Data #endif }; +template +struct AggregateFunctionSingleValueOrNullData : Data +{ + static constexpr bool is_nullable = true; + + using Self = AggregateFunctionSingleValueOrNullData; + + bool first_value = true; + bool is_null = false; + + bool changeIfBetter(const IColumn & column, size_t row_num, Arena * arena) + { + if (first_value) + { + first_value = false; + this->change(column, row_num, arena); + return true; + } + else if (!this->isEqualTo(column, row_num)) + { + is_null = true; + } + return false; + } + + bool changeIfBetter(const Self & to, Arena * arena) + { + if (first_value) + { + first_value = false; + this->change(to, arena); + return true; + } + else if (!this->isEqualTo(to)) + { + is_null = true; + } + return false; + } + + void insertResultInto(IColumn & to) const + { + if (is_null || first_value) + { + to.insertDefault(); + } + else + { + ColumnNullable & col = typeid_cast(to); + col.getNullMapColumn().insertDefault(); + this->Data::insertResultInto(col.getNestedColumn()); + } + } + + static const char * name() { return "singleValueOrNull"; } + +#if USE_EMBEDDED_COMPILER + + static constexpr bool is_compilable = false; + +#endif +}; /** Implement 'heavy hitters' algorithm. * Selects most frequent value if its frequency is more than 50% in each thread of execution. @@ -1074,7 +1143,10 @@ public: DataTypePtr getReturnType() const override { - return this->argument_types.at(0); + auto result_type = this->argument_types.at(0); + if constexpr (Data::is_nullable) + return makeNullable(result_type); + return result_type; } void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override diff --git a/src/AggregateFunctions/AggregateFunctionSingleValueOrNull.cpp b/src/AggregateFunctions/AggregateFunctionSingleValueOrNull.cpp new file mode 100644 index 00000000000..cd897dfcf6e --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionSingleValueOrNull.cpp @@ -0,0 +1,27 @@ +#include +#include +#include +#include "registerAggregateFunctions.h" + + +namespace DB +{ +struct Settings; + +namespace +{ + +AggregateFunctionPtr createAggregateFunctionSingleValueOrNull(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings) +{ + return AggregateFunctionPtr(createAggregateFunctionSingleValue(name, argument_types, parameters, settings)); +} + +} + +void registerAggregateFunctionSingleValueOrNull(AggregateFunctionFactory & factory) +{ + factory.registerFunction("singleValueOrNull", createAggregateFunctionSingleValueOrNull); +} + + +} diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp index 383f10ac24b..70248d4cfde 100644 --- a/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -48,6 +48,7 @@ void registerAggregateFunctionRankCorrelation(AggregateFunctionFactory &); void registerAggregateFunctionMannWhitney(AggregateFunctionFactory &); void registerAggregateFunctionWelchTTest(AggregateFunctionFactory &); void registerAggregateFunctionStudentTTest(AggregateFunctionFactory &); +void registerAggregateFunctionSingleValueOrNull(AggregateFunctionFactory &); void registerAggregateFunctionSequenceNextNode(AggregateFunctionFactory &); class AggregateFunctionCombinatorFactory; @@ -113,6 +114,7 @@ void registerAggregateFunctions() registerAggregateFunctionSequenceNextNode(factory); registerAggregateFunctionWelchTTest(factory); registerAggregateFunctionStudentTTest(factory); + registerAggregateFunctionSingleValueOrNull(factory); registerWindowFunctions(factory); diff --git a/src/Common/DenseHashMap.h b/src/Common/DenseHashMap.h deleted file mode 100644 index 9ac21c82676..00000000000 --- a/src/Common/DenseHashMap.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once -#include - -/// DenseHashMap is a wrapper for google::dense_hash_map. -/// Some hacks are needed to make it work in "Arcadia". -/// "Arcadia" is a proprietary monorepository in Yandex. -/// It uses slightly changed version of sparsehash with a different set of hash functions (which we don't need). -/// Those defines are needed to make it compile. -#if defined(ARCADIA_BUILD) -#define HASH_FUN_H -template -struct THash; -#endif - -#include - -#if !defined(ARCADIA_BUILD) - template , - class EqualKey = std::equal_to, - class Alloc = google::libc_allocator_with_realloc>> - using DenseHashMap = google::dense_hash_map; -#else - template , - class EqualKey = std::equal_to, - class Alloc = google::sparsehash::libc_allocator_with_realloc>> - using DenseHashMap = google::sparsehash::dense_hash_map; - - #undef THash -#endif diff --git a/src/Common/DenseHashSet.h b/src/Common/DenseHashSet.h deleted file mode 100644 index e8c06f36aa3..00000000000 --- a/src/Common/DenseHashSet.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -/// DenseHashSet is a wrapper for google::dense_hash_set. -/// See comment in DenseHashMap.h -#if defined(ARCADIA_BUILD) -#define HASH_FUN_H -template -struct THash; -#endif - -#include - -#if !defined(ARCADIA_BUILD) - template , - class EqualKey = std::equal_to, - class Alloc = google::libc_allocator_with_realloc> - using DenseHashSet = google::dense_hash_set; -#else - template , - class EqualKey = std::equal_to, - class Alloc = google::sparsehash::libc_allocator_with_realloc> - using DenseHashSet = google::sparsehash::dense_hash_set; - - #undef THash -#endif diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 5f7954ed1f7..33f1c616328 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -563,7 +563,8 @@ M(593, ZERO_COPY_REPLICATION_ERROR) \ M(594, BZIP2_STREAM_DECODER_FAILED) \ M(595, BZIP2_STREAM_ENCODER_FAILED) \ - M(596, CANNOT_ADVISE) \ + M(596, INTERSECT_OR_EXCEPT_RESULT_STRUCTURES_MISMATCH) \ + M(597, CANNOT_ADVISE) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Common/SparseHashMap.h b/src/Common/SparseHashMap.h index f01fc633d84..0f86cc13612 100644 --- a/src/Common/SparseHashMap.h +++ b/src/Common/SparseHashMap.h @@ -1,7 +1,6 @@ #pragma once /// SparseHashMap is a wrapper for google::sparse_hash_map. -/// See comment in DenseHashMap.h #if defined(ARCADIA_BUILD) #define HASH_FUN_H template diff --git a/src/Compression/CompressedWriteBuffer.cpp b/src/Compression/CompressedWriteBuffer.cpp index 8d146e8de23..7454e09fda0 100644 --- a/src/Compression/CompressedWriteBuffer.cpp +++ b/src/Compression/CompressedWriteBuffer.cpp @@ -30,16 +30,18 @@ void CompressedWriteBuffer::nextImpl() compressed_buffer.resize(compressed_reserve_size); UInt32 compressed_size = codec->compress(working_buffer.begin(), decompressed_size, compressed_buffer.data()); - // FIXME remove this after fixing msan report in lz4. - // Almost always reproduces on stateless tests, the exact test unknown. - __msan_unpoison(compressed_buffer.data(), compressed_size); - CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(compressed_buffer.data(), compressed_size); out.write(reinterpret_cast(&checksum), CHECKSUM_SIZE); out.write(compressed_buffer.data(), compressed_size); } +void CompressedWriteBuffer::finalize() +{ + next(); +} + + CompressedWriteBuffer::CompressedWriteBuffer( WriteBuffer & out_, CompressionCodecPtr codec_, @@ -48,6 +50,7 @@ CompressedWriteBuffer::CompressedWriteBuffer( { } + CompressedWriteBuffer::~CompressedWriteBuffer() { /// FIXME move final flush into the caller diff --git a/src/Compression/CompressedWriteBuffer.h b/src/Compression/CompressedWriteBuffer.h index a9612b463a5..57ba679855e 100644 --- a/src/Compression/CompressedWriteBuffer.h +++ b/src/Compression/CompressedWriteBuffer.h @@ -29,6 +29,8 @@ public: CompressionCodecPtr codec_ = CompressionCodecFactory::instance().getDefaultCodec(), size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); + void finalize() override; + /// The amount of compressed data size_t getCompressedBytes() { diff --git a/src/Core/NamesAndTypes.cpp b/src/Core/NamesAndTypes.cpp index 54f83fc13fc..b47f5a6823b 100644 --- a/src/Core/NamesAndTypes.cpp +++ b/src/Core/NamesAndTypes.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,7 +7,6 @@ #include #include #include -#include namespace DB @@ -163,8 +163,7 @@ NamesAndTypesList NamesAndTypesList::filter(const Names & names) const NamesAndTypesList NamesAndTypesList::addTypes(const Names & names) const { /// NOTE: It's better to make a map in `IStorage` than to create it here every time again. - DenseHashMap types; - types.set_empty_key(StringRef()); + HashMapWithSavedHash types; for (const auto & column : *this) types[column.name] = &column.type; @@ -172,10 +171,11 @@ NamesAndTypesList NamesAndTypesList::addTypes(const Names & names) const NamesAndTypesList res; for (const String & name : names) { - auto it = types.find(name); + const auto * it = types.find(name); if (it == types.end()) - throw Exception("No column " + name, ErrorCodes::THERE_IS_NO_COLUMN); - res.emplace_back(name, *it->second); + throw Exception(ErrorCodes::THERE_IS_NO_COLUMN, "No column {}", name); + + res.emplace_back(name, *it->getMapped()); } return res; diff --git a/src/DataTypes/DataTypeMap.cpp b/src/DataTypes/DataTypeMap.cpp index 8fd375aa86e..b0bf459b4ca 100644 --- a/src/DataTypes/DataTypeMap.cpp +++ b/src/DataTypes/DataTypeMap.cpp @@ -79,7 +79,7 @@ void DataTypeMap::assertKeyType() const std::string DataTypeMap::doGetName() const { WriteBufferFromOwnString s; - s << "Map(" << key_type->getName() << "," << value_type->getName() << ")"; + s << "Map(" << key_type->getName() << ", " << value_type->getName() << ")"; return s.str(); } diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index a390abc4eaf..d7bbcaf8d46 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -115,6 +115,13 @@ private: [[maybe_unused]] const NullMap * const null_map_data, [[maybe_unused]] const NullMap * const null_map_item) { + if constexpr (std::is_same_v && std::is_same_v) + { + /// Generic variant is using IColumn::compare function that only allows to compare columns of identical types. + if (typeid(data) != typeid(target)) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Columns {} and {} cannot be compared", data.getName(), target.getName()); + } + const size_t size = offsets.size(); result.resize(size); diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 9e6061c2525..e5b381b4d08 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -1,14 +1,17 @@ #include #include #include -#include -#include #include +#include #include #include +#include #include #include +#include +#include #include +#include #include #include #include @@ -24,11 +27,9 @@ #include #include #include -#include -#include #include -#include #include +#include #include #include @@ -44,9 +45,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -65,7 +68,6 @@ #include #include #include -#include #include #include @@ -109,6 +111,10 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMut ProfileEvents::increment(ProfileEvents::SelectQuery); return std::make_unique(query, context, options); } + else if (query->as()) + { + return std::make_unique(query, context, options); + } else if (query->as()) { ProfileEvents::increment(ProfileEvents::InsertQuery); diff --git a/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp b/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp new file mode 100644 index 00000000000..9c8dda56b44 --- /dev/null +++ b/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp @@ -0,0 +1,148 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INTERSECT_OR_EXCEPT_RESULT_STRUCTURES_MISMATCH; + extern const int LOGICAL_ERROR; +} + +static Block getCommonHeader(const Blocks & headers) +{ + size_t num_selects = headers.size(); + Block common_header = headers.front(); + size_t num_columns = common_header.columns(); + + for (size_t query_num = 1; query_num < num_selects; ++query_num) + { + if (headers[query_num].columns() != num_columns) + throw Exception(ErrorCodes::INTERSECT_OR_EXCEPT_RESULT_STRUCTURES_MISMATCH, + "Different number of columns in IntersectExceptQuery elements:\n {} \nand\n {}", + common_header.dumpNames(), headers[query_num].dumpNames()); + } + + std::vector columns(num_selects); + for (size_t column_num = 0; column_num < num_columns; ++column_num) + { + for (size_t i = 0; i < num_selects; ++i) + columns[i] = &headers[i].getByPosition(column_num); + + ColumnWithTypeAndName & result_elem = common_header.getByPosition(column_num); + result_elem = getLeastSuperColumn(columns); + } + + return common_header; +} + +InterpreterSelectIntersectExceptQuery::InterpreterSelectIntersectExceptQuery( + const ASTPtr & query_ptr_, + ContextPtr context_, + const SelectQueryOptions & options_) + : IInterpreterUnionOrSelectQuery(query_ptr_->clone(), context_, options_) +{ + ASTSelectIntersectExceptQuery * ast = query_ptr->as(); + final_operator = ast->final_operator; + + const auto & children = ast->children; + size_t num_children = children.size(); + + /// AST must have been changed by the visitor. + if (final_operator == Operator::UNKNOWN || num_children != 2) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "SelectIntersectExceptyQuery has not been normalized (number of children: {})", + num_children); + + nested_interpreters.resize(num_children); + + for (size_t i = 0; i < num_children; ++i) + nested_interpreters[i] = buildCurrentChildInterpreter(children.at(i)); + + Blocks headers(num_children); + for (size_t query_num = 0; query_num < num_children; ++query_num) + headers[query_num] = nested_interpreters[query_num]->getSampleBlock(); + + result_header = getCommonHeader(headers); +} + +std::unique_ptr +InterpreterSelectIntersectExceptQuery::buildCurrentChildInterpreter(const ASTPtr & ast_ptr_) +{ + if (ast_ptr_->as()) + return std::make_unique(ast_ptr_, context, SelectQueryOptions()); + + if (ast_ptr_->as()) + return std::make_unique(ast_ptr_, context, SelectQueryOptions()); + + if (ast_ptr_->as()) + return std::make_unique(ast_ptr_, context, SelectQueryOptions()); + + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected query: {}", ast_ptr_->getID()); +} + +void InterpreterSelectIntersectExceptQuery::buildQueryPlan(QueryPlan & query_plan) +{ + size_t num_plans = nested_interpreters.size(); + std::vector> plans(num_plans); + DataStreams data_streams(num_plans); + + for (size_t i = 0; i < num_plans; ++i) + { + plans[i] = std::make_unique(); + nested_interpreters[i]->buildQueryPlan(*plans[i]); + + if (!blocksHaveEqualStructure(plans[i]->getCurrentDataStream().header, result_header)) + { + auto actions_dag = ActionsDAG::makeConvertingActions( + plans[i]->getCurrentDataStream().header.getColumnsWithTypeAndName(), + result_header.getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Position); + auto converting_step = std::make_unique(plans[i]->getCurrentDataStream(), std::move(actions_dag)); + converting_step->setStepDescription("Conversion before UNION"); + plans[i]->addStep(std::move(converting_step)); + } + + data_streams[i] = plans[i]->getCurrentDataStream(); + } + + auto max_threads = context->getSettingsRef().max_threads; + auto step = std::make_unique(std::move(data_streams), final_operator, max_threads); + query_plan.unitePlans(std::move(step), std::move(plans)); +} + +BlockIO InterpreterSelectIntersectExceptQuery::execute() +{ + BlockIO res; + + QueryPlan query_plan; + buildQueryPlan(query_plan); + + auto pipeline = query_plan.buildQueryPipeline( + QueryPlanOptimizationSettings::fromContext(context), + BuildQueryPipelineSettings::fromContext(context)); + + res.pipeline = std::move(*pipeline); + res.pipeline.addInterpreterContext(context); + + return res; +} + +void InterpreterSelectIntersectExceptQuery::ignoreWithTotals() +{ + for (auto & interpreter : nested_interpreters) + interpreter->ignoreWithTotals(); +} + +} diff --git a/src/Interpreters/InterpreterSelectIntersectExceptQuery.h b/src/Interpreters/InterpreterSelectIntersectExceptQuery.h new file mode 100644 index 00000000000..805565e4c51 --- /dev/null +++ b/src/Interpreters/InterpreterSelectIntersectExceptQuery.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include +#include + + +namespace DB +{ + +class Context; +class InterpreterSelectQuery; +class QueryPlan; + +class InterpreterSelectIntersectExceptQuery : public IInterpreterUnionOrSelectQuery +{ +using Operator = ASTSelectIntersectExceptQuery::Operator; + +public: + InterpreterSelectIntersectExceptQuery( + const ASTPtr & query_ptr_, + ContextPtr context_, + const SelectQueryOptions & options_); + + BlockIO execute() override; + + Block getSampleBlock() { return result_header; } + + void ignoreWithTotals() override; + +private: + static String getName() { return "SelectIntersectExceptQuery"; } + + std::unique_ptr + buildCurrentChildInterpreter(const ASTPtr & ast_ptr_); + + void buildQueryPlan(QueryPlan & query_plan) override; + + std::vector> nested_interpreters; + + Operator final_operator; +}; + +} diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 3cf4a905d38..cd06f51cb12 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -2,8 +2,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -208,8 +210,10 @@ InterpreterSelectWithUnionQuery::buildCurrentChildInterpreter(const ASTPtr & ast { if (ast_ptr_->as()) return std::make_unique(ast_ptr_, context, options, current_required_result_column_names); - else + else if (ast_ptr_->as()) return std::make_unique(ast_ptr_, context, options, current_required_result_column_names); + else + return std::make_unique(ast_ptr_, context, options); } InterpreterSelectWithUnionQuery::~InterpreterSelectWithUnionQuery() = default; @@ -225,10 +229,14 @@ Block InterpreterSelectWithUnionQuery::getSampleBlock(const ASTPtr & query_ptr_, } if (is_subquery) + { return cache[key] = InterpreterSelectWithUnionQuery(query_ptr_, context_, SelectQueryOptions().subquery().analyze()).getSampleBlock(); + } else + { return cache[key] = InterpreterSelectWithUnionQuery(query_ptr_, context_, SelectQueryOptions().analyze()).getSampleBlock(); + } } diff --git a/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.cpp b/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.cpp index 798c2f2e376..0990667b2a8 100644 --- a/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.cpp +++ b/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.cpp @@ -1,5 +1,6 @@ #include #include +#include #include namespace DB diff --git a/src/Interpreters/SelectIntersectExceptQueryVisitor.cpp b/src/Interpreters/SelectIntersectExceptQueryVisitor.cpp new file mode 100644 index 00000000000..e26c4371591 --- /dev/null +++ b/src/Interpreters/SelectIntersectExceptQueryVisitor.cpp @@ -0,0 +1,129 @@ +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +/* + * Note: there is a difference between intersect and except behaviour. + * `intersect` is supposed to be a part of the last SelectQuery, i.e. the sequence with no parenthesis: + * select 1 union all select 2 except select 1 intersect 2 except select 2 union distinct select 5; + * is interpreted as: + * select 1 union all select 2 except (select 1 intersect 2) except select 2 union distinct select 5; + * Whereas `except` is applied to all left union part like: + * (((select 1 union all select 2) except (select 1 intersect 2)) except select 2) union distinct select 5; +**/ + +void SelectIntersectExceptQueryMatcher::visit(ASTPtr & ast, Data & data) +{ + if (auto * select_union = ast->as()) + visit(*select_union, data); +} + +void SelectIntersectExceptQueryMatcher::visit(ASTSelectWithUnionQuery & ast, Data &) +{ + const auto & union_modes = ast.list_of_modes; + + if (union_modes.empty()) + return; + + auto selects = std::move(ast.list_of_selects->children); + + if (union_modes.size() + 1 != selects.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Incorrect ASTSelectWithUnionQuery (modes: {}, selects: {})", + union_modes.size(), selects.size()); + + std::reverse(selects.begin(), selects.end()); + + ASTs children = {selects.back()}; + selects.pop_back(); + ASTSelectWithUnionQuery::UnionModes modes; + + for (const auto & mode : union_modes) + { + switch (mode) + { + case ASTSelectWithUnionQuery::Mode::EXCEPT: + { + auto left = std::make_shared(); + left->union_mode = ASTSelectWithUnionQuery::Mode::ALL; + + left->list_of_selects = std::make_shared(); + left->children.push_back(left->list_of_selects); + left->list_of_selects->children = std::move(children); + + left->list_of_modes = std::move(modes); + modes = {}; + + auto right = selects.back(); + selects.pop_back(); + + auto except_node = std::make_shared(); + except_node->final_operator = ASTSelectIntersectExceptQuery::Operator::EXCEPT; + except_node->children = {left, right}; + + children = {except_node}; + break; + } + case ASTSelectWithUnionQuery::Mode::INTERSECT: + { + bool from_except = false; + const auto * except_ast = typeid_cast(children.back().get()); + if (except_ast && (except_ast->final_operator == ASTSelectIntersectExceptQuery::Operator::EXCEPT)) + from_except = true; + + ASTPtr left; + if (from_except) + { + left = std::move(children.back()->children[1]); + } + else + { + left = children.back(); + children.pop_back(); + } + + auto right = selects.back(); + selects.pop_back(); + + auto intersect_node = std::make_shared(); + intersect_node->final_operator = ASTSelectIntersectExceptQuery::Operator::INTERSECT; + intersect_node->children = {left, right}; + + if (from_except) + children.back()->children[1] = std::move(intersect_node); + else + children.push_back(std::move(intersect_node)); + + break; + } + default: + { + auto right = selects.back(); + selects.pop_back(); + children.emplace_back(std::move(right)); + modes.push_back(mode); + break; + } + } + } + + if (!selects.empty()) + { + auto right = selects.back(); + selects.pop_back(); + children.emplace_back(std::move(right)); + } + + ast.union_mode = ASTSelectWithUnionQuery::Mode::Unspecified; + ast.list_of_selects->children = std::move(children); + ast.list_of_modes = std::move(modes); +} + +} diff --git a/src/Interpreters/SelectIntersectExceptQueryVisitor.h b/src/Interpreters/SelectIntersectExceptQueryVisitor.h new file mode 100644 index 00000000000..07a6ad606a1 --- /dev/null +++ b/src/Interpreters/SelectIntersectExceptQueryVisitor.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include +#include + +#include +#include + + +namespace DB +{ + +class ASTFunction; + +class SelectIntersectExceptQueryMatcher +{ +public: + struct Data {}; + + static bool needChildVisit(const ASTPtr &, const ASTPtr &) { return true; } + + static void visit(ASTPtr & ast, Data &); + static void visit(ASTSelectWithUnionQuery &, Data &); +}; + +/// Visit children first. +using SelectIntersectExceptQueryVisitor + = InDepthNodeVisitor; +} diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 3ebc2eb142c..23fb35deee3 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include @@ -490,9 +491,16 @@ static std::tuple executeQueryImpl( ApplyWithGlobalVisitor().visit(ast); } - /// Normalize SelectWithUnionQuery - NormalizeSelectWithUnionQueryVisitor::Data data{context->getSettingsRef().union_default_mode}; - NormalizeSelectWithUnionQueryVisitor{data}.visit(ast); + { + SelectIntersectExceptQueryVisitor::Data data; + SelectIntersectExceptQueryVisitor{data}.visit(ast); + } + + { + /// Normalize SelectWithUnionQuery + NormalizeSelectWithUnionQueryVisitor::Data data{context->getSettingsRef().union_default_mode}; + NormalizeSelectWithUnionQueryVisitor{data}.visit(ast); + } /// Check the limits. checkASTSizeLimits(*ast, settings); diff --git a/src/Parsers/ASTSelectIntersectExceptQuery.cpp b/src/Parsers/ASTSelectIntersectExceptQuery.cpp new file mode 100644 index 00000000000..3b9cb0a2c16 --- /dev/null +++ b/src/Parsers/ASTSelectIntersectExceptQuery.cpp @@ -0,0 +1,41 @@ +#include +#include +#include + + +namespace DB +{ + +ASTPtr ASTSelectIntersectExceptQuery::clone() const +{ + auto res = std::make_shared(*this); + + res->children.clear(); + for (const auto & child : children) + res->children.push_back(child->clone()); + + res->final_operator = final_operator; + + cloneOutputOptions(*res); + return res; +} + +void ASTSelectIntersectExceptQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +{ + std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); + + for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it) + { + if (it != children.begin()) + { + settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") + << (final_operator == Operator::INTERSECT ? "INTERSECT" : "EXCEPT") + << (settings.hilite ? hilite_none : "") + << settings.nl_or_ws; + } + + (*it)->formatImpl(settings, state, frame); + } +} + +} diff --git a/src/Parsers/ASTSelectIntersectExceptQuery.h b/src/Parsers/ASTSelectIntersectExceptQuery.h new file mode 100644 index 00000000000..97a8296ce2c --- /dev/null +++ b/src/Parsers/ASTSelectIntersectExceptQuery.h @@ -0,0 +1,31 @@ +#pragma once + +#include + + +namespace DB +{ + +class ASTSelectIntersectExceptQuery : public ASTQueryWithOutput +{ +public: + String getID(char) const override { return "SelectIntersectExceptQuery"; } + + ASTPtr clone() const override; + + void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + + const char * getQueryKindString() const override { return "SelectIntersectExcept"; } + + enum class Operator + { + UNKNOWN, + INTERSECT, + EXCEPT + }; + + /// Final operator after applying visitor. + Operator final_operator = Operator::UNKNOWN; +}; + +} diff --git a/src/Parsers/ASTSelectWithUnionQuery.h b/src/Parsers/ASTSelectWithUnionQuery.h index 0465bdac3a6..629e9b5d96d 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.h +++ b/src/Parsers/ASTSelectWithUnionQuery.h @@ -22,7 +22,9 @@ public: { Unspecified, ALL, - DISTINCT + DISTINCT, + EXCEPT, + INTERSECT }; using UnionModes = std::vector; diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index ca563ddea41..16f2b720b4a 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -1713,6 +1713,8 @@ const char * ParserAlias::restricted_keywords[] = "WHERE", "WINDOW", "WITH", + "INTERSECT", + "EXCEPT", nullptr }; diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 5047ce39d47..58f5e766905 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -1,17 +1,21 @@ #include +#include #include #include #include +#include +#include +#include #include #include +#include #include namespace DB { - const char * ParserMultiplicativeExpression::operators[] = { "*", "multiply", @@ -108,12 +112,18 @@ bool ParserList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) bool ParserUnionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { + ParserUnionQueryElement elem_parser; + ParserKeyword s_union_parser("UNION"); + ParserKeyword s_all_parser("ALL"); + ParserKeyword s_distinct_parser("DISTINCT"); + ParserKeyword s_except_parser("EXCEPT"); + ParserKeyword s_intersect_parser("INTERSECT"); ASTs elements; auto parse_element = [&] { ASTPtr element; - if (!elem_parser->parse(pos, element, expected)) + if (!elem_parser.parse(pos, element, expected)) return false; elements.push_back(element); @@ -123,21 +133,33 @@ bool ParserUnionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) /// Parse UNION type auto parse_separator = [&] { - if (s_union_parser->ignore(pos, expected)) + if (s_union_parser.ignore(pos, expected)) { // SELECT ... UNION ALL SELECT ... - if (s_all_parser->check(pos, expected)) + if (s_all_parser.check(pos, expected)) { union_modes.push_back(ASTSelectWithUnionQuery::Mode::ALL); } // SELECT ... UNION DISTINCT SELECT ... - else if (s_distinct_parser->check(pos, expected)) + else if (s_distinct_parser.check(pos, expected)) { union_modes.push_back(ASTSelectWithUnionQuery::Mode::DISTINCT); } // SELECT ... UNION SELECT ... else + { union_modes.push_back(ASTSelectWithUnionQuery::Mode::Unspecified); + } + return true; + } + else if (s_except_parser.check(pos, expected)) + { + union_modes.push_back(ASTSelectWithUnionQuery::Mode::EXCEPT); + return true; + } + else if (s_intersect_parser.check(pos, expected)) + { + union_modes.push_back(ASTSelectWithUnionQuery::Mode::INTERSECT); return true; } return false; @@ -169,6 +191,91 @@ static bool parseOperator(IParser::Pos & pos, const char * op, Expected & expect } } +enum class SubqueryFunctionType +{ + NONE, + ANY, + ALL +}; + +static bool modifyAST(ASTPtr ast, SubqueryFunctionType type) +{ + /* Rewrite in AST: + * = ANY --> IN + * != ALL --> NOT IN + * = ALL --> IN (SELECT singleValueOrNull(*) FROM subquery) + * != ANY --> NOT IN (SELECT singleValueOrNull(*) FROM subquery) + **/ + + auto * function = assert_cast(ast.get()); + String operator_name = function->name; + + auto function_equals = operator_name == "equals"; + auto function_not_equals = operator_name == "notEquals"; + + String aggregate_function_name; + if (function_equals || function_not_equals) + { + if (operator_name == "notEquals") + function->name = "notIn"; + else + function->name = "in"; + + if ((type == SubqueryFunctionType::ANY && function_equals) + || (type == SubqueryFunctionType::ALL && function_not_equals)) + { + return true; + } + + aggregate_function_name = "singleValueOrNull"; + } + else if (operator_name == "greaterOrEquals" || operator_name == "greater") + { + aggregate_function_name = (type == SubqueryFunctionType::ANY ? "min" : "max"); + } + else if (operator_name == "lessOrEquals" || operator_name == "less") + { + aggregate_function_name = (type == SubqueryFunctionType::ANY ? "max" : "min"); + } + else + return false; + + /// subquery --> (SELECT aggregate_function(*) FROM subquery) + auto aggregate_function = makeASTFunction(aggregate_function_name, std::make_shared()); + auto subquery_node = function->children[0]->children[1]; + + auto table_expression = std::make_shared(); + table_expression->subquery = std::move(subquery_node); + table_expression->children.push_back(table_expression->subquery); + + auto tables_in_select_element = std::make_shared(); + tables_in_select_element->table_expression = std::move(table_expression); + tables_in_select_element->children.push_back(tables_in_select_element->table_expression); + + auto tables_in_select = std::make_shared(); + tables_in_select->children.push_back(std::move(tables_in_select_element)); + + auto select_exp_list = std::make_shared(); + select_exp_list->children.push_back(aggregate_function); + + auto select_query = std::make_shared(); + select_query->children.push_back(select_exp_list); + select_query->children.push_back(tables_in_select); + + select_query->setExpression(ASTSelectQuery::Expression::SELECT, select_exp_list); + select_query->setExpression(ASTSelectQuery::Expression::TABLES, tables_in_select); + + auto select_with_union_query = std::make_shared(); + select_with_union_query->list_of_selects = std::make_shared(); + select_with_union_query->list_of_selects->children.push_back(std::move(select_query)); + select_with_union_query->children.push_back(select_with_union_query->list_of_selects); + + auto new_subquery = std::make_shared(); + new_subquery->children.push_back(select_with_union_query); + ast->children[0]->children.back() = std::move(new_subquery); + + return true; +} bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { @@ -213,7 +320,15 @@ bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, ASTPtr & node auto exp_list = std::make_shared(); ASTPtr elem; - if (!(remaining_elem_parser ? remaining_elem_parser : first_elem_parser)->parse(pos, elem, expected)) + SubqueryFunctionType subquery_function_type = SubqueryFunctionType::NONE; + if (allow_any_all_operators && ParserKeyword("ANY").ignore(pos, expected)) + subquery_function_type = SubqueryFunctionType::ANY; + else if (allow_any_all_operators && ParserKeyword("ALL").ignore(pos, expected)) + subquery_function_type = SubqueryFunctionType::ALL; + else if (!(remaining_elem_parser ? remaining_elem_parser : first_elem_parser)->parse(pos, elem, expected)) + return false; + + if (subquery_function_type != SubqueryFunctionType::NONE && !ParserSubquery().parse(pos, elem, expected)) return false; /// the first argument of the function is the previous element, the second is the next one @@ -224,6 +339,9 @@ bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, ASTPtr & node exp_list->children.push_back(node); exp_list->children.push_back(elem); + if (allow_any_all_operators && subquery_function_type != SubqueryFunctionType::NONE && !modifyAST(function, subquery_function_type)) + return false; + /** special exception for the access operator to the element of the array `x[y]`, which * contains the infix part '[' and the suffix ''] '(specified as' [') */ @@ -855,4 +973,3 @@ bool ParserKeyValuePairsList::parseImpl(Pos & pos, ASTPtr & node, Expected & exp } } - diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index bd4763297d4..17deec4e9e4 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -79,14 +79,6 @@ private: class ParserUnionList : public IParserBase { public: - ParserUnionList(ParserPtr && elem_parser_, ParserPtr && s_union_parser_, ParserPtr && s_all_parser_, ParserPtr && s_distinct_parser_) - : elem_parser(std::move(elem_parser_)) - , s_union_parser(std::move(s_union_parser_)) - , s_all_parser(std::move(s_all_parser_)) - , s_distinct_parser(std::move(s_distinct_parser_)) - { - } - template static bool parseUtil(Pos & pos, const ElemFunc & parse_element, const SepFunc & parse_separator) { @@ -116,10 +108,6 @@ protected: const char * getName() const override { return "list of union elements"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; private: - ParserPtr elem_parser; - ParserPtr s_union_parser; - ParserPtr s_all_parser; - ParserPtr s_distinct_parser; ASTSelectWithUnionQuery::UnionModes union_modes; }; @@ -133,6 +121,8 @@ private: Operators_t overlapping_operators_to_skip = { (const char *[]){ nullptr } }; ParserPtr first_elem_parser; ParserPtr remaining_elem_parser; + /// =, !=, <, > ALL (subquery) / ANY (subquery) + bool allow_any_all_operators = false; public: /** `operators_` - allowed operators and their corresponding functions @@ -142,8 +132,10 @@ public: { } - ParserLeftAssociativeBinaryOperatorList(Operators_t operators_, Operators_t overlapping_operators_to_skip_, ParserPtr && first_elem_parser_) - : operators(operators_), overlapping_operators_to_skip(overlapping_operators_to_skip_), first_elem_parser(std::move(first_elem_parser_)) + ParserLeftAssociativeBinaryOperatorList(Operators_t operators_, + Operators_t overlapping_operators_to_skip_, ParserPtr && first_elem_parser_, bool allow_any_all_operators_ = false) + : operators(operators_), overlapping_operators_to_skip(overlapping_operators_to_skip_), + first_elem_parser(std::move(first_elem_parser_)), allow_any_all_operators(allow_any_all_operators_) { } @@ -353,7 +345,8 @@ class ParserComparisonExpression : public IParserBase private: static const char * operators[]; static const char * overlapping_operators_to_skip[]; - ParserLeftAssociativeBinaryOperatorList operator_parser {operators, overlapping_operators_to_skip, std::make_unique()}; + ParserLeftAssociativeBinaryOperatorList operator_parser {operators, + overlapping_operators_to_skip, std::make_unique(), true}; protected: const char * getName() const override{ return "comparison expression"; } @@ -364,7 +357,6 @@ protected: } }; - /** Parser for nullity checking with IS (NOT) NULL. */ class ParserNullityChecking : public IParserBase diff --git a/src/Parsers/ParserQueryWithOutput.cpp b/src/Parsers/ParserQueryWithOutput.cpp index d5aa1e47533..82f9f561187 100644 --- a/src/Parsers/ParserQueryWithOutput.cpp +++ b/src/Parsers/ParserQueryWithOutput.cpp @@ -1,27 +1,27 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include #include #include -#include +#include +#include +#include +#include #include diff --git a/src/Parsers/ParserSelectWithUnionQuery.cpp b/src/Parsers/ParserSelectWithUnionQuery.cpp index 87e2dab1a47..532a9e20735 100644 --- a/src/Parsers/ParserSelectWithUnionQuery.cpp +++ b/src/Parsers/ParserSelectWithUnionQuery.cpp @@ -10,12 +10,7 @@ namespace DB bool ParserSelectWithUnionQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ASTPtr list_node; - - ParserUnionList parser( - std::make_unique(), - std::make_unique("UNION"), - std::make_unique("ALL"), - std::make_unique("DISTINCT")); + ParserUnionList parser; if (!parser.parse(pos, list_node, expected)) return false; diff --git a/src/Processors/QueryPlan/IntersectOrExceptStep.cpp b/src/Processors/QueryPlan/IntersectOrExceptStep.cpp new file mode 100644 index 00000000000..d1bb1eb41e9 --- /dev/null +++ b/src/Processors/QueryPlan/IntersectOrExceptStep.cpp @@ -0,0 +1,87 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +static Block checkHeaders(const DataStreams & input_streams_) +{ + if (input_streams_.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot perform intersect/except on empty set of query plan steps"); + + Block res = input_streams_.front().header; + for (const auto & stream : input_streams_) + assertBlocksHaveEqualStructure(stream.header, res, "IntersectOrExceptStep"); + + return res; +} + +IntersectOrExceptStep::IntersectOrExceptStep( + DataStreams input_streams_ , Operator operator_ , size_t max_threads_) + : header(checkHeaders(input_streams_)) + , current_operator(operator_) + , max_threads(max_threads_) +{ + input_streams = std::move(input_streams_); + output_stream = DataStream{.header = header}; +} + +QueryPipelinePtr IntersectOrExceptStep::updatePipeline(QueryPipelines pipelines, const BuildQueryPipelineSettings &) +{ + auto pipeline = std::make_unique(); + QueryPipelineProcessorsCollector collector(*pipeline, this); + + if (pipelines.empty()) + { + pipeline->init(Pipe(std::make_shared(output_stream->header))); + processors = collector.detachProcessors(); + return pipeline; + } + + for (auto & cur_pipeline : pipelines) + { + /// Just in case. + if (!isCompatibleHeader(cur_pipeline->getHeader(), getOutputStream().header)) + { + auto converting_dag = ActionsDAG::makeConvertingActions( + cur_pipeline->getHeader().getColumnsWithTypeAndName(), + getOutputStream().header.getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Name); + + auto converting_actions = std::make_shared(std::move(converting_dag)); + cur_pipeline->addSimpleTransform([&](const Block & cur_header) + { + return std::make_shared(cur_header, converting_actions); + }); + } + + /// For the case of union. + cur_pipeline->addTransform(std::make_shared(header, cur_pipeline->getNumStreams(), 1)); + } + + *pipeline = QueryPipeline::unitePipelines(std::move(pipelines), max_threads); + pipeline->addTransform(std::make_shared(header, current_operator)); + + processors = collector.detachProcessors(); + return pipeline; +} + +void IntersectOrExceptStep::describePipeline(FormatSettings & settings) const +{ + IQueryPlanStep::describePipeline(processors, settings); +} + +} diff --git a/src/Processors/QueryPlan/IntersectOrExceptStep.h b/src/Processors/QueryPlan/IntersectOrExceptStep.h new file mode 100644 index 00000000000..9e87c921ab2 --- /dev/null +++ b/src/Processors/QueryPlan/IntersectOrExceptStep.h @@ -0,0 +1,30 @@ +#pragma once +#include +#include + + +namespace DB +{ + +class IntersectOrExceptStep : public IQueryPlanStep +{ +using Operator = ASTSelectIntersectExceptQuery::Operator; + +public: + /// max_threads is used to limit the number of threads for result pipeline. + IntersectOrExceptStep(DataStreams input_streams_, Operator operator_, size_t max_threads_ = 0); + + String getName() const override { return "IntersectOrExcept"; } + + QueryPipelinePtr updatePipeline(QueryPipelines pipelines, const BuildQueryPipelineSettings & settings) override; + + void describePipeline(FormatSettings & settings) const override; + +private: + Block header; + Operator current_operator; + size_t max_threads; + Processors processors; +}; + +} diff --git a/src/Processors/Transforms/IntersectOrExceptTransform.cpp b/src/Processors/Transforms/IntersectOrExceptTransform.cpp new file mode 100644 index 00000000000..3e39123ae4b --- /dev/null +++ b/src/Processors/Transforms/IntersectOrExceptTransform.cpp @@ -0,0 +1,197 @@ +#include + + +namespace DB +{ + +/// After visitor is applied, ASTSelectIntersectExcept always has two child nodes. +IntersectOrExceptTransform::IntersectOrExceptTransform(const Block & header_, Operator operator_) + : IProcessor(InputPorts(2, header_), {header_}) + , current_operator(operator_) +{ + const Names & columns = header_.getNames(); + size_t num_columns = columns.empty() ? header_.columns() : columns.size(); + + key_columns_pos.reserve(columns.size()); + for (size_t i = 0; i < num_columns; ++i) + { + auto pos = columns.empty() ? i : header_.getPositionByName(columns[i]); + key_columns_pos.emplace_back(pos); + } +} + + +IntersectOrExceptTransform::Status IntersectOrExceptTransform::prepare() +{ + auto & output = outputs.front(); + + if (output.isFinished()) + { + for (auto & in : inputs) + in.close(); + + return Status::Finished; + } + + if (!output.canPush()) + { + for (auto & input : inputs) + input.setNotNeeded(); + + return Status::PortFull; + } + + if (current_output_chunk) + { + output.push(std::move(current_output_chunk)); + } + + if (finished_second_input) + { + if (inputs.front().isFinished()) + { + output.finish(); + return Status::Finished; + } + } + else if (inputs.back().isFinished()) + { + finished_second_input = true; + } + + if (!has_input) + { + InputPort & input = finished_second_input ? inputs.front() : inputs.back(); + + input.setNeeded(); + if (!input.hasData()) + return Status::NeedData; + + current_input_chunk = input.pull(); + has_input = true; + } + + return Status::Ready; +} + + +void IntersectOrExceptTransform::work() +{ + if (!finished_second_input) + { + accumulate(std::move(current_input_chunk)); + } + else + { + filter(current_input_chunk); + current_output_chunk = std::move(current_input_chunk); + } + + has_input = false; +} + + +template +void IntersectOrExceptTransform::addToSet(Method & method, const ColumnRawPtrs & columns, size_t rows, SetVariants & variants) const +{ + typename Method::State state(columns, key_sizes, nullptr); + + for (size_t i = 0; i < rows; ++i) + state.emplaceKey(method.data, i, variants.string_pool); +} + + +template +size_t IntersectOrExceptTransform::buildFilter( + Method & method, const ColumnRawPtrs & columns, IColumn::Filter & filter, size_t rows, SetVariants & variants) const +{ + typename Method::State state(columns, key_sizes, nullptr); + size_t new_rows_num = 0; + + for (size_t i = 0; i < rows; ++i) + { + auto find_result = state.findKey(method.data, i, variants.string_pool); + filter[i] = current_operator == ASTSelectIntersectExceptQuery::Operator::EXCEPT ? !find_result.isFound() : find_result.isFound(); + if (filter[i]) + ++new_rows_num; + } + return new_rows_num; +} + + +void IntersectOrExceptTransform::accumulate(Chunk chunk) +{ + auto num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + + ColumnRawPtrs column_ptrs; + column_ptrs.reserve(key_columns_pos.size()); + + for (auto pos : key_columns_pos) + column_ptrs.emplace_back(columns[pos].get()); + + if (!data) + data.emplace(); + + if (data->empty()) + data->init(SetVariants::chooseMethod(column_ptrs, key_sizes)); + + auto & data_set = *data; + switch (data->type) + { + case SetVariants::Type::EMPTY: + break; +#define M(NAME) \ + case SetVariants::Type::NAME: \ + addToSet(*data_set.NAME, column_ptrs, num_rows, data_set); \ + break; + APPLY_FOR_SET_VARIANTS(M) +#undef M + } +} + + +void IntersectOrExceptTransform::filter(Chunk & chunk) +{ + auto num_rows = chunk.getNumRows(); + auto columns = chunk.detachColumns(); + + ColumnRawPtrs column_ptrs; + column_ptrs.reserve(key_columns_pos.size()); + + for (auto pos : key_columns_pos) + column_ptrs.emplace_back(columns[pos].get()); + + if (!data) + data.emplace(); + + if (data->empty()) + data->init(SetVariants::chooseMethod(column_ptrs, key_sizes)); + + size_t new_rows_num = 0; + + IColumn::Filter filter(num_rows); + auto & data_set = *data; + + switch (data->type) + { + case SetVariants::Type::EMPTY: + break; +#define M(NAME) \ + case SetVariants::Type::NAME: \ + new_rows_num = buildFilter(*data_set.NAME, column_ptrs, filter, num_rows, data_set); \ + break; + APPLY_FOR_SET_VARIANTS(M) +#undef M + } + + if (!new_rows_num) + return; + + for (auto & column : columns) + column = column->filter(filter, -1); + + chunk.setColumns(std::move(columns), new_rows_num); +} + +} diff --git a/src/Processors/Transforms/IntersectOrExceptTransform.h b/src/Processors/Transforms/IntersectOrExceptTransform.h new file mode 100644 index 00000000000..e200bfd6cc5 --- /dev/null +++ b/src/Processors/Transforms/IntersectOrExceptTransform.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ + +class IntersectOrExceptTransform : public IProcessor +{ +using Operator = ASTSelectIntersectExceptQuery::Operator; + +public: + IntersectOrExceptTransform(const Block & header_, Operator operator_); + + String getName() const override { return "IntersectOrExcept"; } + +protected: + Status prepare() override; + + void work() override; + +private: + Operator current_operator; + + ColumnNumbers key_columns_pos; + std::optional data; + Sizes key_sizes; + + Chunk current_input_chunk; + Chunk current_output_chunk; + + bool finished_second_input = false; + bool has_input = false; + + void accumulate(Chunk chunk); + + void filter(Chunk & chunk); + + template + void addToSet(Method & method, const ColumnRawPtrs & key_columns, size_t rows, SetVariants & variants) const; + + template + size_t buildFilter(Method & method, const ColumnRawPtrs & columns, + IColumn::Filter & filter, size_t rows, SetVariants & variants) const; +}; + +} diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 5378b84a5d0..d659259e1a9 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -48,7 +48,6 @@ IMergeTreeReader::IMergeTreeReader( part_columns = Nested::collect(part_columns); } - columns_from_part.set_empty_key(StringRef()); for (const auto & column_from_part : part_columns) columns_from_part[column_from_part.name] = &column_from_part.type; } @@ -213,7 +212,7 @@ NameAndTypePair IMergeTreeReader::getColumnFromPart(const NameAndTypePair & requ { auto name_in_storage = required_column.getNameInStorage(); - decltype(columns_from_part.begin()) it; + ColumnsFromPart::ConstLookupResult it; if (alter_conversions.isColumnRenamed(name_in_storage)) { String old_name = alter_conversions.getColumnOldName(name_in_storage); @@ -227,7 +226,7 @@ NameAndTypePair IMergeTreeReader::getColumnFromPart(const NameAndTypePair & requ if (it == columns_from_part.end()) return required_column; - const auto & type = *it->second; + const DataTypePtr & type = *it->getMapped(); if (required_column.isSubcolumn()) { auto subcolumn_name = required_column.getSubcolumnName(); @@ -236,10 +235,10 @@ NameAndTypePair IMergeTreeReader::getColumnFromPart(const NameAndTypePair & requ if (!subcolumn_type) return required_column; - return {String(it->first), subcolumn_name, type, subcolumn_type}; + return {String(it->getKey()), subcolumn_name, type, subcolumn_type}; } - return {String(it->first), type}; + return {String(it->getKey()), type}; } void IMergeTreeReader::performRequiredConversions(Columns & res_columns) diff --git a/src/Storages/MergeTree/IMergeTreeReader.h b/src/Storages/MergeTree/IMergeTreeReader.h index 8d80719efaf..696cc2f105b 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.h +++ b/src/Storages/MergeTree/IMergeTreeReader.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include @@ -95,7 +95,8 @@ private: /// Actual data type of columns in part - DenseHashMap columns_from_part; + using ColumnsFromPart = HashMapWithSavedHash; + ColumnsFromPart columns_from_part; }; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index ea5f7cfc36a..ef276a53df2 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1012,8 +1012,24 @@ bool ReplicatedMergeTreeQueue::isNotCoveredByFuturePartsImpl(const String & log_ bool ReplicatedMergeTreeQueue::addFuturePartIfNotCoveredByThem(const String & part_name, LogEntry & entry, String & reject_reason) { + /// We have found `part_name` on some replica and are going to fetch it instead of covered `entry->new_part_name`. std::lock_guard lock(state_mutex); + if (virtual_parts.getContainingPart(part_name).empty()) + { + /// We should not fetch any parts that absent in our `virtual_parts` set, + /// because we do not know about such parts according to our replication queue (we know about them from some side-channel). + /// Otherwise, it may break invariants in replication queue reordering, for example: + /// 1. Our queue contains GET_PART all_2_2_0, log contains DROP_RANGE all_2_2_0 and MERGE_PARTS all_1_3_1 + /// 2. We execute GET_PART all_2_2_0, but fetch all_1_3_1 instead + /// (drop_ranges.isAffectedByDropRange(...) is false-negative, because DROP_RANGE all_2_2_0 is not pulled yet). + /// It actually means, that MERGE_PARTS all_1_3_1 is executed too, but it's not even pulled yet. + /// 3. Then we pull log, trying to execute DROP_RANGE all_2_2_0 + /// and reveal that it was incorrectly reordered with MERGE_PARTS all_1_3_1 (drop range intersects merged part). + reject_reason = fmt::format("Log entry for part {} or covering part is not pulled from log to queue yet.", part_name); + return false; + } + /// FIXME get rid of actual_part_name. /// If new covering part jumps over DROP_RANGE we should execute drop range first if (drop_ranges.isAffectedByDropRange(part_name, reject_reason)) diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 91f69cdac7d..5183b925141 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -1,7 +1,7 @@ #include -#include -#include +#include +#include #include #include #include @@ -320,8 +320,7 @@ Block StorageInMemoryMetadata::getSampleBlockForColumns( { Block res; - DenseHashMap virtuals_map; - virtuals_map.set_empty_key(StringRef()); + HashMapWithSavedHash virtuals_map; /// Virtual columns must be appended after ordinary, because user can /// override them. @@ -335,9 +334,9 @@ Block StorageInMemoryMetadata::getSampleBlockForColumns( { res.insert({column->type->createColumn(), column->type, column->name}); } - else if (auto it = virtuals_map.find(name); it != virtuals_map.end()) + else if (auto * it = virtuals_map.find(name); it != virtuals_map.end()) { - const auto & type = *it->second; + const auto & type = *it->getMapped(); res.insert({type->createColumn(), type, name}); } else @@ -470,8 +469,8 @@ bool StorageInMemoryMetadata::hasSelectQuery() const namespace { - using NamesAndTypesMap = DenseHashMap; - using UniqueStrings = DenseHashSet; + using NamesAndTypesMap = HashMapWithSavedHash; + using UniqueStrings = HashSetWithSavedHash; String listOfColumns(const NamesAndTypesList & available_columns) { @@ -488,20 +487,12 @@ namespace NamesAndTypesMap getColumnsMap(const NamesAndTypesList & columns) { NamesAndTypesMap res; - res.set_empty_key(StringRef()); for (const auto & column : columns) res.insert({column.name, column.type.get()}); return res; } - - UniqueStrings initUniqueStrings() - { - UniqueStrings strings; - strings.set_empty_key(StringRef()); - return strings; - } } void StorageInMemoryMetadata::check(const Names & column_names, const NamesAndTypesList & virtuals, const StorageID & storage_id) const @@ -514,11 +505,12 @@ void StorageInMemoryMetadata::check(const Names & column_names, const NamesAndTy } const auto virtuals_map = getColumnsMap(virtuals); - auto unique_names = initUniqueStrings(); + UniqueStrings unique_names; for (const auto & name : column_names) { - bool has_column = getColumns().hasColumnOrSubcolumn(ColumnsDescription::AllPhysical, name) || virtuals_map.count(name); + bool has_column = getColumns().hasColumnOrSubcolumn(ColumnsDescription::AllPhysical, name) + || virtuals_map.find(name) != nullptr; if (!has_column) { @@ -540,23 +532,31 @@ void StorageInMemoryMetadata::check(const NamesAndTypesList & provided_columns) const NamesAndTypesList & available_columns = getColumns().getAllPhysical(); const auto columns_map = getColumnsMap(available_columns); - auto unique_names = initUniqueStrings(); + UniqueStrings unique_names; + for (const NameAndTypePair & column : provided_columns) { - auto it = columns_map.find(column.name); + const auto * it = columns_map.find(column.name); if (columns_map.end() == it) throw Exception( - "There is no column with name " + column.name + ". There are columns: " + listOfColumns(available_columns), - ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); + ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, + "There is no column with name {}. There are columns: {}", + column.name, + listOfColumns(available_columns)); - if (!column.type->equals(*it->second)) + if (!column.type->equals(*it->getMapped())) throw Exception( - "Type mismatch for column " + column.name + ". Column has type " + it->second->getName() + ", got type " - + column.type->getName(), - ErrorCodes::TYPE_MISMATCH); + ErrorCodes::TYPE_MISMATCH, + "Type mismatch for column {}. Column has type {}, got type {}", + column.name, + it->getMapped()->getName(), + column.type->getName()); if (unique_names.end() != unique_names.find(column.name)) - throw Exception("Column " + column.name + " queried more than once", ErrorCodes::COLUMN_QUERIED_MORE_THAN_ONCE); + throw Exception(ErrorCodes::COLUMN_QUERIED_MORE_THAN_ONCE, + "Column {} queried more than once", + column.name); + unique_names.insert(column.name); } } @@ -572,26 +572,38 @@ void StorageInMemoryMetadata::check(const NamesAndTypesList & provided_columns, "Empty list of columns queried. There are columns: " + listOfColumns(available_columns), ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED); - auto unique_names = initUniqueStrings(); + UniqueStrings unique_names; + for (const String & name : column_names) { - auto it = provided_columns_map.find(name); + const auto * it = provided_columns_map.find(name); if (provided_columns_map.end() == it) continue; - auto jt = available_columns_map.find(name); + const auto * jt = available_columns_map.find(name); if (available_columns_map.end() == jt) throw Exception( - "There is no column with name " + name + ". There are columns: " + listOfColumns(available_columns), - ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); + ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, + "There is no column with name {}. There are columns: {}", + name, + listOfColumns(available_columns)); - if (!it->second->equals(*jt->second)) + const auto & provided_column_type = *it->getMapped(); + const auto & available_column_type = *jt->getMapped(); + + if (!provided_column_type.equals(available_column_type)) throw Exception( - "Type mismatch for column " + name + ". Column has type " + jt->second->getName() + ", got type " + it->second->getName(), - ErrorCodes::TYPE_MISMATCH); + ErrorCodes::TYPE_MISMATCH, + "Type mismatch for column {}. Column has type {}, got type {}", + name, + provided_column_type.getName(), + available_column_type.getName()); if (unique_names.end() != unique_names.find(name)) - throw Exception("Column " + name + " queried more than once", ErrorCodes::COLUMN_QUERIED_MORE_THAN_ONCE); + throw Exception(ErrorCodes::COLUMN_QUERIED_MORE_THAN_ONCE, + "Column {} queried more than once", + name); + unique_names.insert(name); } } @@ -612,17 +624,21 @@ void StorageInMemoryMetadata::check(const Block & block, bool need_all) const names_in_block.insert(column.name); - auto it = columns_map.find(column.name); + const auto * it = columns_map.find(column.name); if (columns_map.end() == it) throw Exception( - "There is no column with name " + column.name + ". There are columns: " + listOfColumns(available_columns), - ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); + ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, + "There is no column with name {}. There are columns: {}", + column.name, + listOfColumns(available_columns)); - if (!column.type->equals(*it->second)) + if (!column.type->equals(*it->getMapped())) throw Exception( - "Type mismatch for column " + column.name + ". Column has type " + it->second->getName() + ", got type " - + column.type->getName(), - ErrorCodes::TYPE_MISMATCH); + ErrorCodes::TYPE_MISMATCH, + "Type mismatch for column {}. Column has type {}, got type {}", + column.name, + it->getMapped()->getName(), + column.type->getName()); } if (need_all && names_in_block.size() < columns_map.size()) diff --git a/tests/queries/0_stateless/01318_map_add_map_subtract_on_map_type.reference b/tests/queries/0_stateless/01318_map_add_map_subtract_on_map_type.reference index 96bafc2c79c..de34b856130 100644 --- a/tests/queries/0_stateless/01318_map_add_map_subtract_on_map_type.reference +++ b/tests/queries/0_stateless/01318_map_add_map_subtract_on_map_type.reference @@ -18,38 +18,38 @@ {1:3,2:2,8:2} {1:3,2:2,9:2} {1:3,2:2,10:2} -{1:2,2:2} Map(UInt8,UInt64) -{1:2,2:2} Map(UInt16,UInt64) -{1:2,2:2} Map(UInt32,UInt64) -{1:2,2:2} Map(UInt64,UInt64) -{1:2,2:2} Map(UInt128,UInt128) -{1:2,2:2} Map(UInt256,UInt256) -{1:2,2:2} Map(Int16,UInt64) -{1:2,2:2} Map(Int16,Int64) -{1:2,2:2} Map(Int32,Int64) -{1:2,2:2} Map(Int64,Int64) -{1:2,2:2} Map(Int128,Int128) -{1:2,2:2} Map(Int256,Int256) -{1:3.300000023841858,2:2} Map(UInt8,Float64) -{1:3.3000000000000003,2:2} Map(UInt8,Float64) +{1:2,2:2} Map(UInt8, UInt64) +{1:2,2:2} Map(UInt16, UInt64) +{1:2,2:2} Map(UInt32, UInt64) +{1:2,2:2} Map(UInt64, UInt64) +{1:2,2:2} Map(UInt128, UInt128) +{1:2,2:2} Map(UInt256, UInt256) +{1:2,2:2} Map(Int16, UInt64) +{1:2,2:2} Map(Int16, Int64) +{1:2,2:2} Map(Int32, Int64) +{1:2,2:2} Map(Int64, Int64) +{1:2,2:2} Map(Int128, Int128) +{1:2,2:2} Map(Int256, Int256) +{1:3.300000023841858,2:2} Map(UInt8, Float64) +{1:3.3000000000000003,2:2} Map(UInt8, Float64) {'a':1,'b':2} {'a':1,'b':1,'c':1} {'a':1,'b':1,'d':1} -{'a':1,'b':2} Map(String,UInt64) -{'a':1,'b':1,'c':1} Map(String,UInt64) -{'a':1,'b':1,'d':1} Map(String,UInt64) +{'a':1,'b':2} Map(String, UInt64) +{'a':1,'b':1,'c':1} Map(String, UInt64) +{'a':1,'b':1,'d':1} Map(String, UInt64) {'a':1,'b':2} {'a':1,'b':1,'c':1} {'a':1,'b':1,'d':1} -{'a':2} Map(Enum16(\'a\' = 1, \'b\' = 2),Int64) -{'b':2} Map(Enum16(\'a\' = 1, \'b\' = 2),Int64) -{'a':2} Map(Enum8(\'a\' = 1, \'b\' = 2),Int64) -{'b':2} Map(Enum8(\'a\' = 1, \'b\' = 2),Int64) -{'00000000-89ab-cdef-0123-456789abcdef':2} Map(UUID,Int64) -{'11111111-89ab-cdef-0123-456789abcdef':4} Map(UUID,Int64) -{1:0,2:0} Map(UInt8,UInt64) -{1:18446744073709551615,2:18446744073709551615} Map(UInt8,UInt64) -{1:-1,2:-1} Map(UInt8,Int64) -{1:-1.0999999761581423,2:0} Map(UInt8,Float64) -{1:-1,2:-1} Map(UInt8,Int64) -{1:-2,2:-2,3:1} Map(UInt8,Int64) +{'a':2} Map(Enum16(\'a\' = 1, \'b\' = 2), Int64) +{'b':2} Map(Enum16(\'a\' = 1, \'b\' = 2), Int64) +{'a':2} Map(Enum8(\'a\' = 1, \'b\' = 2), Int64) +{'b':2} Map(Enum8(\'a\' = 1, \'b\' = 2), Int64) +{'00000000-89ab-cdef-0123-456789abcdef':2} Map(UUID, Int64) +{'11111111-89ab-cdef-0123-456789abcdef':4} Map(UUID, Int64) +{1:0,2:0} Map(UInt8, UInt64) +{1:18446744073709551615,2:18446744073709551615} Map(UInt8, UInt64) +{1:-1,2:-1} Map(UInt8, Int64) +{1:-1.0999999761581423,2:0} Map(UInt8, Float64) +{1:-1,2:-1} Map(UInt8, Int64) +{1:-2,2:-2,3:1} Map(UInt8, Int64) diff --git a/tests/queries/0_stateless/01550_type_map_formats.reference b/tests/queries/0_stateless/01550_type_map_formats.reference index ca081db75a2..998473ef63a 100644 --- a/tests/queries/0_stateless/01550_type_map_formats.reference +++ b/tests/queries/0_stateless/01550_type_map_formats.reference @@ -4,15 +4,15 @@ JSON [ { "name": "m", - "type": "Map(String,UInt32)" + "type": "Map(String, UInt32)" }, { "name": "m1", - "type": "Map(String,Date)" + "type": "Map(String, Date)" }, { "name": "m2", - "type": "Map(String,Array(UInt32))" + "type": "Map(String, Array(UInt32))" } ], diff --git a/tests/queries/0_stateless/01925_map_populate_series_on_map.reference b/tests/queries/0_stateless/01925_map_populate_series_on_map.reference index 235a227f548..fd3d3b2450d 100644 --- a/tests/queries/0_stateless/01925_map_populate_series_on_map.reference +++ b/tests/queries/0_stateless/01925_map_populate_series_on_map.reference @@ -29,39 +29,39 @@ select mapPopulateSeries(m, n) from map_test; {1:1,2:0,3:0,4:0,5:2,6:0} drop table map_test; select mapPopulateSeries(map(toUInt8(1), toUInt8(1), 2, 1)) as res, toTypeName(res); -{1:1,2:1} Map(UInt8,UInt8) +{1:1,2:1} Map(UInt8, UInt8) select mapPopulateSeries(map(toUInt16(1), toUInt16(1), 2, 1)) as res, toTypeName(res); -{1:1,2:1} Map(UInt16,UInt16) +{1:1,2:1} Map(UInt16, UInt16) select mapPopulateSeries(map(toUInt32(1), toUInt32(1), 2, 1)) as res, toTypeName(res); -{1:1,2:1} Map(UInt32,UInt32) +{1:1,2:1} Map(UInt32, UInt32) select mapPopulateSeries(map(toUInt64(1), toUInt64(1), 2, 1)) as res, toTypeName(res); -{1:1,2:1} Map(UInt64,UInt64) +{1:1,2:1} Map(UInt64, UInt64) select mapPopulateSeries(map(toUInt128(1), toUInt128(1), 2, 1)) as res, toTypeName(res); -{1:1,2:1} Map(UInt128,UInt128) +{1:1,2:1} Map(UInt128, UInt128) select mapPopulateSeries(map(toUInt256(1), toUInt256(1), 2, 1)) as res, toTypeName(res); -{1:1,2:1} Map(UInt256,UInt256) +{1:1,2:1} Map(UInt256, UInt256) select mapPopulateSeries(map(toInt8(1), toInt8(1), 2, 1)) as res, toTypeName(res); -{1:1,2:1} Map(Int16,Int16) +{1:1,2:1} Map(Int16, Int16) select mapPopulateSeries(map(toInt16(1), toInt16(1), 2, 1)) as res, toTypeName(res); -{1:1,2:1} Map(Int16,Int16) +{1:1,2:1} Map(Int16, Int16) select mapPopulateSeries(map(toInt32(1), toInt32(1), 2, 1)) as res, toTypeName(res); -{1:1,2:1} Map(Int32,Int32) +{1:1,2:1} Map(Int32, Int32) select mapPopulateSeries(map(toInt64(1), toInt64(1), 2, 1)) as res, toTypeName(res); -{1:1,2:1} Map(Int64,Int64) +{1:1,2:1} Map(Int64, Int64) select mapPopulateSeries(map(toInt128(1), toInt128(1), 2, 1)) as res, toTypeName(res); -{1:1,2:1} Map(Int128,Int128) +{1:1,2:1} Map(Int128, Int128) select mapPopulateSeries(map(toInt256(1), toInt256(1), 2, 1)) as res, toTypeName(res); -{1:1,2:1} Map(Int256,Int256) +{1:1,2:1} Map(Int256, Int256) select mapPopulateSeries(map(toInt8(-10), toInt8(1), 2, 1)) as res, toTypeName(res); -{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int16,Int16) +{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int16, Int16) select mapPopulateSeries(map(toInt16(-10), toInt16(1), 2, 1)) as res, toTypeName(res); -{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int16,Int16) +{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int16, Int16) select mapPopulateSeries(map(toInt32(-10), toInt32(1), 2, 1)) as res, toTypeName(res); -{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int32,Int32) +{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int32, Int32) select mapPopulateSeries(map(toInt64(-10), toInt64(1), 2, 1)) as res, toTypeName(res); -{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int64,Int64) +{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0,-4:0,-3:0,-2:0,-1:0,0:0,1:0,2:1} Map(Int64, Int64) select mapPopulateSeries(map(toInt64(-10), toInt64(1), 2, 1), toInt64(-5)) as res, toTypeName(res); -{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0} Map(Int64,Int64) +{-10:1,-9:0,-8:0,-7:0,-6:0,-5:0} Map(Int64, Int64) select mapPopulateSeries(); -- { serverError 42 } select mapPopulateSeries('asdf'); -- { serverError 43 } select mapPopulateSeries(map('1', 1, '2', 1)) as res, toTypeName(res); -- { serverError 43 } diff --git a/tests/queries/0_stateless/02002_parse_map_int_key.reference b/tests/queries/0_stateless/02002_parse_map_int_key.reference new file mode 100644 index 00000000000..dc02589d4bc --- /dev/null +++ b/tests/queries/0_stateless/02002_parse_map_int_key.reference @@ -0,0 +1 @@ +{1:2,3:4,5:6,7:8} {'2021-05-20':1,'2021-05-21':2,'2021-05-22':3,'2021-05-23':4} diff --git a/tests/queries/0_stateless/02002_parse_map_int_key.sql b/tests/queries/0_stateless/02002_parse_map_int_key.sql new file mode 100644 index 00000000000..ecd2a090975 --- /dev/null +++ b/tests/queries/0_stateless/02002_parse_map_int_key.sql @@ -0,0 +1,11 @@ +SET allow_experimental_map_type = 1; + +DROP TABLE IF EXISTS t_map_int_key; +CREATE TABLE t_map_int_key (m1 Map(UInt32, UInt32), m2 Map(Date, UInt32)) ENGINE = Memory; + +INSERT INTO t_map_int_key FORMAT CSV "{1:2, 3: 4, 5 :6, 7 : 8}","{'2021-05-20':1, '2021-05-21': 2, '2021-05-22' :3, '2021-05-23' : 4}" +; + +SELECT m1, m2 FROM t_map_int_key; + +DROP TABLE t_map_int_key; diff --git a/tests/queries/0_stateless/02004_intersect_except_operators.reference b/tests/queries/0_stateless/02004_intersect_except_operators.reference new file mode 100644 index 00000000000..03b881f690b --- /dev/null +++ b/tests/queries/0_stateless/02004_intersect_except_operators.reference @@ -0,0 +1,129 @@ +-- { echo } +select 1 intersect select 1; +1 +select 2 intersect select 1; +select 1 except select 1; +select 2 except select 1; +2 +select number from numbers(20) intersect select number from numbers(5, 5); +5 +6 +7 +8 +9 +select number from numbers(10) except select number from numbers(5); +5 +6 +7 +8 +9 +select number, number+10 from numbers(12) except select number+5, number+15 from numbers(10); +0 10 +1 11 +2 12 +3 13 +4 14 +select 1 except select 2 intersect select 1; +1 +select 1 except select 2 intersect select 2; +1 +select 1 intersect select 1 except select 2; +1 +select 1 intersect select 1 except select 1; +select 1 intersect select 1 except select 2 intersect select 1 except select 3 intersect select 1; +1 +select 1 intersect select 1 except select 2 intersect select 1 except select 3 intersect select 2; +1 +select 1 intersect select 1 except select 2 intersect select 1 except select 3 intersect select 2 except select 1; +select number from numbers(10) except select 5; +0 +1 +2 +3 +4 +6 +7 +8 +9 +select number from numbers(100) intersect select number from numbers(20, 60) except select number from numbers(30, 20) except select number from numbers(60, 20); +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +select * from (select 1 intersect select 1); +1 +with (select number from numbers(10) intersect select 5) as a select a * 10; +50 +with (select 5 except select 1) as a select a except select 5; +with (select number from numbers(10) intersect select 5) as a select a intersect select 1; +with (select number from numbers(10) intersect select 5) as a select a except select 1; +5 +select count() from (select number from numbers(10) except select 5); +9 +select count() from (select number from numbers(1000000) intersect select number from numbers(200000, 600000)); +600000 +select count() from (select number from numbers(100) intersect select number from numbers(20, 60) except select number from numbers(30, 20) except select number from numbers(60, 20)); +20 +select count() from (select number from numbers(100) intersect select number from numbers(20, 60) except select number from numbers(30, 20) except select number from numbers(60, 20) union all select number from numbers(100, 10)); +30 +select count() from (select number from numbers(1000000) intersect select number from numbers(200000, 600000) except select number from numbers(300000, 200000) except select number from numbers(600000, 200000)); +200000 +select 1 union all select 1 intersect select 1; +1 +1 +select 1 union all select 1 intersect select 2; +1 +select * from (select 1 union all select 2 union all select 3 union all select 4 except select 3 union all select 5) order by 1; +1 +2 +4 +5 +select * from (select 1 union all select 2 union all select 3 union all select 4 intersect select 3 union all select 5) order by 1; +1 +2 +3 +5 +select * from (select 1 union all select 2 union all select 3 union all select 4 intersect select 3 union all select 5 except select 1) order by 1; +2 +3 +5 +select 1 intersect (select 1 except select 2); +1 +select 1 union all select 2 except (select 2 except select 1 union all select 1) except select 4; +select 1 intersect select count() from (select 1 except select 2 intersect select 2 union all select 1); +explain syntax select 1 intersect select 1; +SELECT 1 +INTERSECT +SELECT 1 +explain syntax select 1 except select 1; +SELECT 1 +EXCEPT +SELECT 1 +explain syntax select 1 union all select 2 except (select 2 except select 1 union all select 1) except select 4; +SELECT 1 +UNION ALL +SELECT 2 +EXCEPT +SELECT 2 +EXCEPT +SELECT 1 +UNION ALL +SELECT 1 +EXCEPT +SELECT 4 diff --git a/tests/queries/0_stateless/02004_intersect_except_operators.sql b/tests/queries/0_stateless/02004_intersect_except_operators.sql new file mode 100644 index 00000000000..7f08cc0adf2 --- /dev/null +++ b/tests/queries/0_stateless/02004_intersect_except_operators.sql @@ -0,0 +1,45 @@ +-- { echo } +select 1 intersect select 1; +select 2 intersect select 1; +select 1 except select 1; +select 2 except select 1; + +select number from numbers(20) intersect select number from numbers(5, 5); +select number from numbers(10) except select number from numbers(5); +select number, number+10 from numbers(12) except select number+5, number+15 from numbers(10); + +select 1 except select 2 intersect select 1; +select 1 except select 2 intersect select 2; +select 1 intersect select 1 except select 2; +select 1 intersect select 1 except select 1; +select 1 intersect select 1 except select 2 intersect select 1 except select 3 intersect select 1; +select 1 intersect select 1 except select 2 intersect select 1 except select 3 intersect select 2; +select 1 intersect select 1 except select 2 intersect select 1 except select 3 intersect select 2 except select 1; + +select number from numbers(10) except select 5; +select number from numbers(100) intersect select number from numbers(20, 60) except select number from numbers(30, 20) except select number from numbers(60, 20); + +select * from (select 1 intersect select 1); +with (select number from numbers(10) intersect select 5) as a select a * 10; +with (select 5 except select 1) as a select a except select 5; +with (select number from numbers(10) intersect select 5) as a select a intersect select 1; +with (select number from numbers(10) intersect select 5) as a select a except select 1; +select count() from (select number from numbers(10) except select 5); +select count() from (select number from numbers(1000000) intersect select number from numbers(200000, 600000)); +select count() from (select number from numbers(100) intersect select number from numbers(20, 60) except select number from numbers(30, 20) except select number from numbers(60, 20)); +select count() from (select number from numbers(100) intersect select number from numbers(20, 60) except select number from numbers(30, 20) except select number from numbers(60, 20) union all select number from numbers(100, 10)); +select count() from (select number from numbers(1000000) intersect select number from numbers(200000, 600000) except select number from numbers(300000, 200000) except select number from numbers(600000, 200000)); + +select 1 union all select 1 intersect select 1; +select 1 union all select 1 intersect select 2; +select * from (select 1 union all select 2 union all select 3 union all select 4 except select 3 union all select 5) order by 1; +select * from (select 1 union all select 2 union all select 3 union all select 4 intersect select 3 union all select 5) order by 1; +select * from (select 1 union all select 2 union all select 3 union all select 4 intersect select 3 union all select 5 except select 1) order by 1; + +select 1 intersect (select 1 except select 2); +select 1 union all select 2 except (select 2 except select 1 union all select 1) except select 4; +select 1 intersect select count() from (select 1 except select 2 intersect select 2 union all select 1); + +explain syntax select 1 intersect select 1; +explain syntax select 1 except select 1; +explain syntax select 1 union all select 2 except (select 2 except select 1 union all select 1) except select 4; diff --git a/tests/queries/0_stateless/02007_test_any_all_operators.reference b/tests/queries/0_stateless/02007_test_any_all_operators.reference new file mode 100644 index 00000000000..a232320d15c --- /dev/null +++ b/tests/queries/0_stateless/02007_test_any_all_operators.reference @@ -0,0 +1,51 @@ +-- { echo } +select 1 == any (select number from numbers(10)); +1 +select 1 == any (select number from numbers(2, 10)); +0 +select 1 != all (select 1 from numbers(10)); +0 +select 1 != all (select number from numbers(10)); +0 +select 1 == all (select 1 from numbers(10)); +1 +select 1 == all (select number from numbers(10)); +0 +select 1 != any (select 1 from numbers(10)); +0 +select 1 != any (select number from numbers(10)); +1 +select number as a from numbers(10) where a == any (select number from numbers(3, 3)); +3 +4 +5 +select number as a from numbers(10) where a != any (select 5 from numbers(3, 3)); +0 +1 +2 +3 +4 +6 +7 +8 +9 +select 1 < any (select 1 from numbers(10)); +0 +select 1 <= any (select 1 from numbers(10)); +1 +select 1 < any (select number from numbers(10)); +1 +select 1 > any (select number from numbers(10)); +1 +select 1 >= any (select number from numbers(10)); +1 +select 11 > all (select number from numbers(10)); +1 +select 11 <= all (select number from numbers(11)); +0 +select 11 < all (select 11 from numbers(10)); +0 +select 11 > all (select 11 from numbers(10)); +0 +select 11 >= all (select 11 from numbers(10)); +1 diff --git a/tests/queries/0_stateless/02007_test_any_all_operators.sql b/tests/queries/0_stateless/02007_test_any_all_operators.sql new file mode 100644 index 00000000000..10d7325afca --- /dev/null +++ b/tests/queries/0_stateless/02007_test_any_all_operators.sql @@ -0,0 +1,26 @@ +-- { echo } +select 1 == any (select number from numbers(10)); +select 1 == any (select number from numbers(2, 10)); + +select 1 != all (select 1 from numbers(10)); +select 1 != all (select number from numbers(10)); + +select 1 == all (select 1 from numbers(10)); +select 1 == all (select number from numbers(10)); + +select 1 != any (select 1 from numbers(10)); +select 1 != any (select number from numbers(10)); + +select number as a from numbers(10) where a == any (select number from numbers(3, 3)); +select number as a from numbers(10) where a != any (select 5 from numbers(3, 3)); + +select 1 < any (select 1 from numbers(10)); +select 1 <= any (select 1 from numbers(10)); +select 1 < any (select number from numbers(10)); +select 1 > any (select number from numbers(10)); +select 1 >= any (select number from numbers(10)); +select 11 > all (select number from numbers(10)); +select 11 <= all (select number from numbers(11)); +select 11 < all (select 11 from numbers(10)); +select 11 > all (select 11 from numbers(10)); +select 11 >= all (select 11 from numbers(10)); diff --git a/tests/queries/0_stateless/02010_array_index_bad_cast.reference b/tests/queries/0_stateless/02010_array_index_bad_cast.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02010_array_index_bad_cast.sql b/tests/queries/0_stateless/02010_array_index_bad_cast.sql new file mode 100644 index 00000000000..19c58bb28a7 --- /dev/null +++ b/tests/queries/0_stateless/02010_array_index_bad_cast.sql @@ -0,0 +1,2 @@ +-- This query throws exception about uncomparable data types (but at least it does not introduce bad cast in code). +SELECT has(materialize(CAST(['2021-07-14'] AS Array(LowCardinality(Nullable(DateTime))))), materialize('2021-07-14'::DateTime64(7))); -- { serverError 44 }