diff --git a/.github/ISSUE_TEMPLATE/40_bug-report.md b/.github/ISSUE_TEMPLATE/40_bug-report.md index 4dfd19266d0..97137366189 100644 --- a/.github/ISSUE_TEMPLATE/40_bug-report.md +++ b/.github/ISSUE_TEMPLATE/40_bug-report.md @@ -7,7 +7,7 @@ assignees: '' --- -(you don't have to strictly follow this form) +You have to provide the following information whenever possible. **Describe the bug** A clear and concise description of what works not as it is supposed to. diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ba56594e08..cb91f879d5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,6 +167,7 @@ endif () # If turned `ON`, assumes the user has either the system GTest library or the bundled one. option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON) +option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF) if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0") # Only for Linux, x86_64. diff --git a/contrib/boost b/contrib/boost index a8d43d3142c..9f0ff347e50 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit a8d43d3142cc6b26fc55bec33f7f6edb1156ab7a +Subproject commit 9f0ff347e50429686604002d8ad1fd07515c4f31 diff --git a/contrib/cyrus-sasl b/contrib/cyrus-sasl index 9995bf9d8e1..e6466edfd63 160000 --- a/contrib/cyrus-sasl +++ b/contrib/cyrus-sasl @@ -1 +1 @@ -Subproject commit 9995bf9d8e14f58934d9313ac64f13780d6dd3c9 +Subproject commit e6466edfd638cc5073debe941c53345b18a09512 diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index f2c2a56cb85..093629e61fc 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -1019,6 +1019,7 @@ done wait # Create per-query flamegraphs +touch report/query-files.txt IFS=$'\n' for version in {right,left} do @@ -1209,6 +1210,12 @@ function upload_results /^metric/ { print old_sha, new_sha, $2, $3 }' \ | "${client[@]}" --query "INSERT INTO run_attributes_v1 FORMAT TSV" + # Grepping numactl results from log is too crazy, I'll just call it again. + "${client[@]}" --query "INSERT INTO run_attributes_v1 FORMAT TSV" <merge(place, rhs, arena); } + void mergeBatch( + size_t batch_size, + AggregateDataPtr * places, + size_t place_offset, + const AggregateDataPtr * rhs, + Arena * arena) const override + { + nested_func->mergeBatch(batch_size, places, place_offset, rhs, arena); + } + void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf) const override { nested_func->serialize(place, buf); diff --git a/src/AggregateFunctions/AggregateFunctionOrFill.h b/src/AggregateFunctions/AggregateFunctionOrFill.h index 4bb25e0d4de..732e83e5a0c 100644 --- a/src/AggregateFunctions/AggregateFunctionOrFill.h +++ b/src/AggregateFunctions/AggregateFunctionOrFill.h @@ -196,6 +196,18 @@ public: place[size_of_data] |= rhs[size_of_data]; } + void mergeBatch( + size_t batch_size, + AggregateDataPtr * places, + size_t place_offset, + const AggregateDataPtr * rhs, + Arena * arena) const override + { + nested_function->mergeBatch(batch_size, places, place_offset, rhs, arena); + for (size_t i = 0; i < batch_size; ++i) + (places[i] + place_offset)[size_of_data] |= rhs[i][size_of_data]; + } + void serialize( ConstAggregateDataPtr place, WriteBuffer & buf) const override diff --git a/src/AggregateFunctions/CMakeLists.txt b/src/AggregateFunctions/CMakeLists.txt index 72cae5428a0..b6dbf2b4eb0 100644 --- a/src/AggregateFunctions/CMakeLists.txt +++ b/src/AggregateFunctions/CMakeLists.txt @@ -24,6 +24,6 @@ list(REMOVE_ITEM clickhouse_aggregate_functions_headers add_library(clickhouse_aggregate_functions ${clickhouse_aggregate_functions_sources}) target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms PUBLIC ${CITYHASH_LIBRARIES}) -if(ENABLE_TESTS) - add_subdirectory(tests) +if(ENABLE_EXAMPLES) + add_subdirectory(examples) endif() diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index a418cdb5523..68c176d14e9 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -153,6 +153,13 @@ public: Arena * arena, ssize_t if_argument_pos = -1) const = 0; + virtual void mergeBatch( + size_t batch_size, + AggregateDataPtr * places, + size_t place_offset, + const AggregateDataPtr * rhs, + Arena * arena) const = 0; + /** The same for single place. */ virtual void addBatchSinglePlace( @@ -279,6 +286,18 @@ public: } } + void mergeBatch( + size_t batch_size, + AggregateDataPtr * places, + size_t place_offset, + const AggregateDataPtr * rhs, + Arena * arena) const override + { + for (size_t i = 0; i < batch_size; ++i) + if (places[i]) + static_cast(this)->merge(places[i] + place_offset, rhs[i], arena); + } + void addBatchSinglePlace( size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1) const override { diff --git a/src/AggregateFunctions/StatCommon.h b/src/AggregateFunctions/StatCommon.h index ba887567ad8..dc94a6b6e77 100644 --- a/src/AggregateFunctions/StatCommon.h +++ b/src/AggregateFunctions/StatCommon.h @@ -11,6 +11,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + template static Float64 integrateSimpson(Float64 a, Float64 b, F && func) { @@ -48,6 +53,11 @@ std::pair computeRanksAndTieCorrection(const Values & value ++right; auto adjusted = (left + right + 1.) / 2.; auto count_equal = right - left; + + /// Scipy implementation throws exception in this case too. + if (count_equal == size) + throw Exception("All numbers in both samples are identical", ErrorCodes::BAD_ARGUMENTS); + tie_numenator += std::pow(count_equal, 3) - count_equal; for (size_t iter = left; iter < right; ++iter) out[indexes[iter]] = adjusted; diff --git a/src/AggregateFunctions/tests/CMakeLists.txt b/src/AggregateFunctions/examples/CMakeLists.txt similarity index 100% rename from src/AggregateFunctions/tests/CMakeLists.txt rename to src/AggregateFunctions/examples/CMakeLists.txt diff --git a/src/AggregateFunctions/tests/quantile-t-digest.cpp b/src/AggregateFunctions/examples/quantile-t-digest.cpp similarity index 100% rename from src/AggregateFunctions/tests/quantile-t-digest.cpp rename to src/AggregateFunctions/examples/quantile-t-digest.cpp diff --git a/src/Client/CMakeLists.txt b/src/Client/CMakeLists.txt index 88c05163602..119414a8a70 100644 --- a/src/Client/CMakeLists.txt +++ b/src/Client/CMakeLists.txt @@ -1 +1,3 @@ -add_subdirectory(tests) +if (ENABLE_EXAMPLES) + add_subdirectory(examples) +endif() \ No newline at end of file diff --git a/src/Client/tests/CMakeLists.txt b/src/Client/examples/CMakeLists.txt similarity index 100% rename from src/Client/tests/CMakeLists.txt rename to src/Client/examples/CMakeLists.txt diff --git a/src/Client/tests/test_connect.cpp b/src/Client/examples/test_connect.cpp similarity index 100% rename from src/Client/tests/test_connect.cpp rename to src/Client/examples/test_connect.cpp diff --git a/src/Columns/CMakeLists.txt b/src/Columns/CMakeLists.txt index 65172356645..f676f415eea 100644 --- a/src/Columns/CMakeLists.txt +++ b/src/Columns/CMakeLists.txt @@ -1,3 +1,3 @@ -if (ENABLE_TESTS) - add_subdirectory (tests) +if (ENABLE_EXAMPLES) + add_subdirectory (examples) endif () diff --git a/src/Columns/tests/CMakeLists.txt b/src/Columns/examples/CMakeLists.txt similarity index 100% rename from src/Columns/tests/CMakeLists.txt rename to src/Columns/examples/CMakeLists.txt diff --git a/src/Common/CMakeLists.txt b/src/Common/CMakeLists.txt index 61d9b9771a4..1935fe4fed1 100644 --- a/src/Common/CMakeLists.txt +++ b/src/Common/CMakeLists.txt @@ -3,6 +3,6 @@ add_subdirectory(StringUtils) #add_subdirectory(ZooKeeper) #add_subdirectory(ConfigProcessor) -if (ENABLE_TESTS) - add_subdirectory (tests) -endif () +if (ENABLE_EXAMPLES) + add_subdirectory(examples) +endif() diff --git a/src/Common/JSONBuilder.cpp b/src/Common/JSONBuilder.cpp new file mode 100644 index 00000000000..30bec88003e --- /dev/null +++ b/src/Common/JSONBuilder.cpp @@ -0,0 +1,99 @@ +#include +#include +#include + +namespace DB::JSONBuilder +{ + +static bool isArrayOrMap(const IItem & item) +{ + return typeid_cast(&item) || typeid_cast(&item); +} + +static bool isSimpleArray(const std::vector & values) +{ + for (const auto & value : values) + if (isArrayOrMap(*value)) + return false; + + return true; +} + +void JSONString::format(const FormatSettings & settings, FormatContext & context) +{ + writeJSONString(value, context.out, settings.settings); +} + +void JSONBool::format(const FormatSettings &, FormatContext & context) +{ + writeString(value ? "true" : "false", context.out); +} + +void JSONArray::format(const FormatSettings & settings, FormatContext & context) +{ + writeChar('[', context.out); + + context.offset += settings.indent; + + bool single_row = settings.print_simple_arrays_in_single_row && isSimpleArray(values); + bool first = true; + + for (const auto & value : values) + { + if (!first) + writeChar(',', context.out); + + if (!single_row) + { + writeChar('\n', context.out); + writeChar(' ', context.offset, context.out); + } + else if (!first) + writeChar(' ', context.out); + + first = false; + value->format(settings, context); + } + + context.offset -= settings.indent; + + if (!single_row) + { + writeChar('\n', context.out); + writeChar(' ', context.offset, context.out); + } + + writeChar(']', context.out); +} + +void JSONMap::format(const FormatSettings & settings, FormatContext & context) +{ + writeChar('{', context.out); + + context.offset += settings.indent; + + bool first = true; + + for (const auto & value : values) + { + if (!first) + writeChar(',', context.out); + first = false; + + writeChar('\n', context.out); + writeChar(' ', context.offset, context.out); + writeJSONString(value.key, context.out, settings.settings); + + writeChar(':', context.out); + writeChar(' ', context.out); + value.value->format(settings, context); + } + + context.offset -= settings.indent; + + writeChar('\n', context.out); + writeChar(' ', context.offset, context.out); + writeChar('}', context.out); +} + +} diff --git a/src/Common/JSONBuilder.h b/src/Common/JSONBuilder.h new file mode 100644 index 00000000000..be95323de7d --- /dev/null +++ b/src/Common/JSONBuilder.h @@ -0,0 +1,111 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace DB::JSONBuilder +{ + +struct FormatSettings +{ + const DB::FormatSettings & settings; + size_t indent = 2; + bool print_simple_arrays_in_single_row = true; +}; + +struct FormatContext +{ + WriteBuffer & out; + size_t offset = 0; +}; + +class IItem +{ +public: + virtual ~IItem() = default; + virtual void format(const FormatSettings & settings, FormatContext & context) = 0; +}; + +using ItemPtr = std::unique_ptr; + +class JSONString : public IItem +{ +public: + explicit JSONString(std::string value_) : value(std::move(value_)) {} + void format(const FormatSettings & settings, FormatContext & context) override; + +private: + std::string value; +}; + +template +class JSONNumber : public IItem +{ +public: + explicit JSONNumber(T value_) : value(value_) + { + static_assert(std::is_arithmetic_v, "JSONNumber support only numeric types"); + } + + void format(const FormatSettings & settings, FormatContext & context) override + { + writeJSONNumber(value, context.out, settings.settings); + } + +private: + T value; +}; + +class JSONBool : public IItem +{ +public: + explicit JSONBool(bool value_) : value(std::move(value_)) {} + void format(const FormatSettings & settings, FormatContext & context) override; + +private: + bool value; +}; + +class JSONArray : public IItem +{ +public: + void add(ItemPtr value) { values.push_back(std::move(value)); } + void add(std::string value) { add(std::make_unique(std::move(value))); } + void add(const char * value) { add(std::make_unique(value)); } + void add(bool value) { add(std::make_unique(std::move(value))); } + + template ::value, bool> = true> + void add(T value) { add(std::make_unique>(value)); } + + void format(const FormatSettings & settings, FormatContext & context) override; + +private: + std::vector values; +}; + +class JSONMap : public IItem +{ + struct Pair + { + std::string key; + ItemPtr value; + }; + +public: + void add(std::string key, ItemPtr value) { values.emplace_back(Pair{.key = std::move(key), .value = std::move(value)}); } + void add(std::string key, std::string value) { add(std::move(key), std::make_unique(std::move(value))); } + void add(std::string key, const char * value) { add(std::move(key), std::make_unique(value)); } + void add(std::string key, bool value) { add(std::move(key), std::make_unique(std::move(value))); } + + template ::value, bool> = true> + void add(std::string key, T value) { add(std::move(key), std::make_unique>(value)); } + + void format(const FormatSettings & settings, FormatContext & context) override; + +private: + std::vector values; +}; + +} diff --git a/src/Common/StatusInfo.cpp b/src/Common/StatusInfo.cpp index be343acfe4a..32afc833001 100644 --- a/src/Common/StatusInfo.cpp +++ b/src/Common/StatusInfo.cpp @@ -1,5 +1,5 @@ #include -#include +#include /// Available status. Add something here as you wish. #define APPLY_FOR_STATUS(M) \ diff --git a/src/Common/ZooKeeper/CMakeLists.txt b/src/Common/ZooKeeper/CMakeLists.txt index adc1ad41e78..d29fba53277 100644 --- a/src/Common/ZooKeeper/CMakeLists.txt +++ b/src/Common/ZooKeeper/CMakeLists.txt @@ -6,6 +6,6 @@ add_library(clickhouse_common_zookeeper ${clickhouse_common_zookeeper_headers} $ target_link_libraries (clickhouse_common_zookeeper PUBLIC clickhouse_common_io common PRIVATE string_utils) -if (ENABLE_TESTS) - add_subdirectory (tests) -endif () +if (ENABLE_EXAMPLES) + add_subdirectory(examples) +endif() diff --git a/src/Common/ZooKeeper/tests/CMakeLists.txt b/src/Common/ZooKeeper/examples/CMakeLists.txt similarity index 100% rename from src/Common/ZooKeeper/tests/CMakeLists.txt rename to src/Common/ZooKeeper/examples/CMakeLists.txt diff --git a/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp b/src/Common/ZooKeeper/examples/zk_many_watches_reconnect.cpp similarity index 100% rename from src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp rename to src/Common/ZooKeeper/examples/zk_many_watches_reconnect.cpp diff --git a/src/Common/ZooKeeper/tests/zkutil_test_async.cpp b/src/Common/ZooKeeper/examples/zkutil_test_async.cpp similarity index 100% rename from src/Common/ZooKeeper/tests/zkutil_test_async.cpp rename to src/Common/ZooKeeper/examples/zkutil_test_async.cpp diff --git a/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp b/src/Common/ZooKeeper/examples/zkutil_test_commands.cpp similarity index 100% rename from src/Common/ZooKeeper/tests/zkutil_test_commands.cpp rename to src/Common/ZooKeeper/examples/zkutil_test_commands.cpp diff --git a/src/Common/ZooKeeper/tests/zkutil_test_commands_new_lib.cpp b/src/Common/ZooKeeper/examples/zkutil_test_commands_new_lib.cpp similarity index 100% rename from src/Common/ZooKeeper/tests/zkutil_test_commands_new_lib.cpp rename to src/Common/ZooKeeper/examples/zkutil_test_commands_new_lib.cpp diff --git a/src/Common/ZooKeeper/tests/zookeeper_impl.cpp b/src/Common/ZooKeeper/examples/zookeeper_impl.cpp similarity index 100% rename from src/Common/ZooKeeper/tests/zookeeper_impl.cpp rename to src/Common/ZooKeeper/examples/zookeeper_impl.cpp diff --git a/src/Common/tests/CMakeLists.txt b/src/Common/examples/CMakeLists.txt similarity index 96% rename from src/Common/tests/CMakeLists.txt rename to src/Common/examples/CMakeLists.txt index 2dd56e862f0..030c18df803 100644 --- a/src/Common/tests/CMakeLists.txt +++ b/src/Common/examples/CMakeLists.txt @@ -35,9 +35,6 @@ target_include_directories(radix_sort SYSTEM PRIVATE ${PDQSORT_INCLUDE_DIR}) add_executable (arena_with_free_lists arena_with_free_lists.cpp) target_link_libraries (arena_with_free_lists PRIVATE dbms) -add_executable (pod_array pod_array.cpp) -target_link_libraries (pod_array PRIVATE clickhouse_common_io) - add_executable (lru_hash_map_perf lru_hash_map_perf.cpp) target_link_libraries (lru_hash_map_perf PRIVATE clickhouse_common_io) diff --git a/src/Common/tests/arena_with_free_lists.cpp b/src/Common/examples/arena_with_free_lists.cpp similarity index 100% rename from src/Common/tests/arena_with_free_lists.cpp rename to src/Common/examples/arena_with_free_lists.cpp diff --git a/src/Common/tests/array_cache.cpp b/src/Common/examples/array_cache.cpp similarity index 100% rename from src/Common/tests/array_cache.cpp rename to src/Common/examples/array_cache.cpp diff --git a/src/Common/tests/auto_array.cpp b/src/Common/examples/auto_array.cpp similarity index 100% rename from src/Common/tests/auto_array.cpp rename to src/Common/examples/auto_array.cpp diff --git a/src/Common/tests/average.cpp b/src/Common/examples/average.cpp similarity index 100% rename from src/Common/tests/average.cpp rename to src/Common/examples/average.cpp diff --git a/src/Common/tests/chaos_sanitizer.cpp b/src/Common/examples/chaos_sanitizer.cpp similarity index 100% rename from src/Common/tests/chaos_sanitizer.cpp rename to src/Common/examples/chaos_sanitizer.cpp diff --git a/src/Common/tests/compact_array.cpp b/src/Common/examples/compact_array.cpp similarity index 100% rename from src/Common/tests/compact_array.cpp rename to src/Common/examples/compact_array.cpp diff --git a/src/Common/tests/cow_columns.cpp b/src/Common/examples/cow_columns.cpp similarity index 100% rename from src/Common/tests/cow_columns.cpp rename to src/Common/examples/cow_columns.cpp diff --git a/src/Common/tests/cow_compositions.cpp b/src/Common/examples/cow_compositions.cpp similarity index 100% rename from src/Common/tests/cow_compositions.cpp rename to src/Common/examples/cow_compositions.cpp diff --git a/src/Common/tests/hashes_test.cpp b/src/Common/examples/hashes_test.cpp similarity index 100% rename from src/Common/tests/hashes_test.cpp rename to src/Common/examples/hashes_test.cpp diff --git a/src/Common/tests/int_hashes_perf.cpp b/src/Common/examples/int_hashes_perf.cpp similarity index 100% rename from src/Common/tests/int_hashes_perf.cpp rename to src/Common/examples/int_hashes_perf.cpp diff --git a/src/Common/tests/integer_hash_tables_and_hashes.cpp b/src/Common/examples/integer_hash_tables_and_hashes.cpp similarity index 100% rename from src/Common/tests/integer_hash_tables_and_hashes.cpp rename to src/Common/examples/integer_hash_tables_and_hashes.cpp diff --git a/src/Common/tests/lru_hash_map_perf.cpp b/src/Common/examples/lru_hash_map_perf.cpp similarity index 100% rename from src/Common/tests/lru_hash_map_perf.cpp rename to src/Common/examples/lru_hash_map_perf.cpp diff --git a/src/Common/tests/memory_statistics_os_perf.cpp b/src/Common/examples/memory_statistics_os_perf.cpp similarity index 100% rename from src/Common/tests/memory_statistics_os_perf.cpp rename to src/Common/examples/memory_statistics_os_perf.cpp diff --git a/src/Common/tests/parallel_aggregation.cpp b/src/Common/examples/parallel_aggregation.cpp similarity index 100% rename from src/Common/tests/parallel_aggregation.cpp rename to src/Common/examples/parallel_aggregation.cpp diff --git a/src/Common/tests/parallel_aggregation2.cpp b/src/Common/examples/parallel_aggregation2.cpp similarity index 100% rename from src/Common/tests/parallel_aggregation2.cpp rename to src/Common/examples/parallel_aggregation2.cpp diff --git a/src/Common/tests/procfs_metrics_provider_perf.cpp b/src/Common/examples/procfs_metrics_provider_perf.cpp similarity index 100% rename from src/Common/tests/procfs_metrics_provider_perf.cpp rename to src/Common/examples/procfs_metrics_provider_perf.cpp diff --git a/src/Common/tests/radix_sort.cpp b/src/Common/examples/radix_sort.cpp similarity index 100% rename from src/Common/tests/radix_sort.cpp rename to src/Common/examples/radix_sort.cpp diff --git a/src/Common/tests/shell_command_inout.cpp b/src/Common/examples/shell_command_inout.cpp similarity index 100% rename from src/Common/tests/shell_command_inout.cpp rename to src/Common/examples/shell_command_inout.cpp diff --git a/src/Common/tests/simple_cache.cpp b/src/Common/examples/simple_cache.cpp similarity index 100% rename from src/Common/tests/simple_cache.cpp rename to src/Common/examples/simple_cache.cpp diff --git a/src/Common/tests/sip_hash_perf.cpp b/src/Common/examples/sip_hash_perf.cpp similarity index 100% rename from src/Common/tests/sip_hash_perf.cpp rename to src/Common/examples/sip_hash_perf.cpp diff --git a/src/Common/tests/small_table.cpp b/src/Common/examples/small_table.cpp similarity index 100% rename from src/Common/tests/small_table.cpp rename to src/Common/examples/small_table.cpp diff --git a/src/Common/tests/space_saving.cpp b/src/Common/examples/space_saving.cpp similarity index 100% rename from src/Common/tests/space_saving.cpp rename to src/Common/examples/space_saving.cpp diff --git a/src/Common/tests/stopwatch.cpp b/src/Common/examples/stopwatch.cpp similarity index 100% rename from src/Common/tests/stopwatch.cpp rename to src/Common/examples/stopwatch.cpp diff --git a/src/Common/tests/symbol_index.cpp b/src/Common/examples/symbol_index.cpp similarity index 100% rename from src/Common/tests/symbol_index.cpp rename to src/Common/examples/symbol_index.cpp diff --git a/src/Common/tests/thread_creation_latency.cpp b/src/Common/examples/thread_creation_latency.cpp similarity index 100% rename from src/Common/tests/thread_creation_latency.cpp rename to src/Common/examples/thread_creation_latency.cpp diff --git a/src/Common/tests/gtest_pod_array.cpp b/src/Common/tests/gtest_pod_array.cpp index 9cc77b88195..2f41f641b09 100644 --- a/src/Common/tests/gtest_pod_array.cpp +++ b/src/Common/tests/gtest_pod_array.cpp @@ -4,6 +4,419 @@ using namespace DB; + +TEST(Common, PODArrayBasicMove) +{ + using namespace DB; + + static constexpr size_t initial_bytes = 32; + using Array = PODArray, initial_bytes>>; + + { + Array arr; + Array arr2; + arr2 = std::move(arr); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + + Array arr2; + + arr2 = std::move(arr); + + ASSERT_EQ(arr2.size(), 3); + ASSERT_EQ(arr2[0], 1); + ASSERT_EQ(arr2[1], 2); + ASSERT_EQ(arr2[2], 3); + + arr = std::move(arr2); + + ASSERT_EQ(arr.size(), 3); + ASSERT_EQ(arr[0], 1); + ASSERT_EQ(arr[1], 2); + ASSERT_EQ(arr[2], 3); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + arr.push_back(4); + arr.push_back(5); + + Array arr2; + + arr2 = std::move(arr); + + ASSERT_EQ(arr2.size(), 5); + ASSERT_EQ(arr2[0], 1); + ASSERT_EQ(arr2[1], 2); + ASSERT_EQ(arr2[2], 3); + ASSERT_EQ(arr2[3], 4); + ASSERT_EQ(arr2[4], 5); + + arr = std::move(arr2); + + ASSERT_EQ(arr.size(), 5); + ASSERT_EQ(arr[0], 1); + ASSERT_EQ(arr[1], 2); + ASSERT_EQ(arr[2], 3); + ASSERT_EQ(arr[3], 4); + ASSERT_EQ(arr[4], 5); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + + Array arr2; + + arr2.push_back(4); + arr2.push_back(5); + arr2.push_back(6); + arr2.push_back(7); + + arr2 = std::move(arr); + + ASSERT_EQ(arr2.size(), 3); + ASSERT_EQ(arr2[0], 1); + ASSERT_EQ(arr2[1], 2); + ASSERT_EQ(arr2[2], 3); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + + Array arr2; + + arr2.push_back(4); + arr2.push_back(5); + arr2.push_back(6); + arr2.push_back(7); + arr2.push_back(8); + + arr = std::move(arr2); + + ASSERT_EQ(arr.size(), 5); + ASSERT_EQ(arr[0], 4); + ASSERT_EQ(arr[1], 5); + ASSERT_EQ(arr[2], 6); + ASSERT_EQ(arr[3], 7); + ASSERT_EQ(arr[4], 8); + } +} + + +TEST(Common, PODArrayBasicSwap) +{ + using namespace DB; + + static constexpr size_t initial_bytes = 32; + using Array = PODArray, initial_bytes>>; + + { + Array arr; + Array arr2; + arr.swap(arr2); + arr2.swap(arr); + } + + { + Array arr; + + Array arr2; + + arr2.push_back(1); + arr2.push_back(2); + arr2.push_back(3); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 3); + ASSERT_TRUE(arr[0] == 1); + ASSERT_TRUE(arr[1] == 2); + ASSERT_TRUE(arr[2] == 3); + + ASSERT_TRUE(arr2.empty()); + + arr.swap(arr2); + + ASSERT_TRUE(arr.empty()); + + ASSERT_TRUE(arr2.size() == 3); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + } + + { + Array arr; + + Array arr2; + + arr2.push_back(1); + arr2.push_back(2); + arr2.push_back(3); + arr2.push_back(4); + arr2.push_back(5); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 5); + ASSERT_TRUE(arr[0] == 1); + ASSERT_TRUE(arr[1] == 2); + ASSERT_TRUE(arr[2] == 3); + ASSERT_TRUE(arr[3] == 4); + ASSERT_TRUE(arr[4] == 5); + + ASSERT_TRUE(arr2.empty()); + + arr.swap(arr2); + + ASSERT_TRUE(arr.empty()); + + ASSERT_TRUE(arr2.size() == 5); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + ASSERT_TRUE(arr2[3] == 4); + ASSERT_TRUE(arr2[4] == 5); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + + Array arr2; + + arr2.push_back(4); + arr2.push_back(5); + arr2.push_back(6); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 3); + ASSERT_TRUE(arr[0] == 4); + ASSERT_TRUE(arr[1] == 5); + ASSERT_TRUE(arr[2] == 6); + + ASSERT_TRUE(arr2.size() == 3); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 3); + ASSERT_TRUE(arr[0] == 1); + ASSERT_TRUE(arr[1] == 2); + ASSERT_TRUE(arr[2] == 3); + + ASSERT_TRUE(arr2.size() == 3); + ASSERT_TRUE(arr2[0] == 4); + ASSERT_TRUE(arr2[1] == 5); + ASSERT_TRUE(arr2[2] == 6); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + + Array arr2; + + arr2.push_back(3); + arr2.push_back(4); + arr2.push_back(5); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 3); + ASSERT_TRUE(arr[0] == 3); + ASSERT_TRUE(arr[1] == 4); + ASSERT_TRUE(arr[2] == 5); + + ASSERT_TRUE(arr2.size() == 2); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 2); + ASSERT_TRUE(arr[0] == 1); + ASSERT_TRUE(arr[1] == 2); + + ASSERT_TRUE(arr2.size() == 3); + ASSERT_TRUE(arr2[0] == 3); + ASSERT_TRUE(arr2[1] == 4); + ASSERT_TRUE(arr2[2] == 5); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + + Array arr2; + + arr2.push_back(4); + arr2.push_back(5); + arr2.push_back(6); + arr2.push_back(7); + arr2.push_back(8); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 5); + ASSERT_TRUE(arr[0] == 4); + ASSERT_TRUE(arr[1] == 5); + ASSERT_TRUE(arr[2] == 6); + ASSERT_TRUE(arr[3] == 7); + ASSERT_TRUE(arr[4] == 8); + + ASSERT_TRUE(arr2.size() == 3); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 3); + ASSERT_TRUE(arr[0] == 1); + ASSERT_TRUE(arr[1] == 2); + ASSERT_TRUE(arr[2] == 3); + + ASSERT_TRUE(arr2.size() == 5); + ASSERT_TRUE(arr2[0] == 4); + ASSERT_TRUE(arr2[1] == 5); + ASSERT_TRUE(arr2[2] == 6); + ASSERT_TRUE(arr2[3] == 7); + ASSERT_TRUE(arr2[4] == 8); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + arr.push_back(4); + arr.push_back(5); + + Array arr2; + + arr2.push_back(6); + arr2.push_back(7); + arr2.push_back(8); + arr2.push_back(9); + arr2.push_back(10); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 5); + ASSERT_TRUE(arr[0] == 6); + ASSERT_TRUE(arr[1] == 7); + ASSERT_TRUE(arr[2] == 8); + ASSERT_TRUE(arr[3] == 9); + ASSERT_TRUE(arr[4] == 10); + + ASSERT_TRUE(arr2.size() == 5); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + ASSERT_TRUE(arr2[3] == 4); + ASSERT_TRUE(arr2[4] == 5); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 5); + ASSERT_TRUE(arr[0] == 1); + ASSERT_TRUE(arr[1] == 2); + ASSERT_TRUE(arr[2] == 3); + ASSERT_TRUE(arr[3] == 4); + ASSERT_TRUE(arr[4] == 5); + + ASSERT_TRUE(arr2.size() == 5); + ASSERT_TRUE(arr2[0] == 6); + ASSERT_TRUE(arr2[1] == 7); + ASSERT_TRUE(arr2[2] == 8); + ASSERT_TRUE(arr2[3] == 9); + ASSERT_TRUE(arr2[4] == 10); + } +} + +TEST(Common, PODArrayBasicSwapMoveConstructor) +{ + static constexpr size_t initial_bytes = 32; + using Array = PODArray, initial_bytes>>; + + { + Array arr; + Array arr2{std::move(arr)}; + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + + Array arr2{std::move(arr)}; + + ASSERT_TRUE(arr.empty()); // NOLINT + + ASSERT_TRUE(arr2.size() == 3); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + arr.push_back(4); + arr.push_back(5); + + Array arr2{std::move(arr)}; + + ASSERT_TRUE(arr.empty()); // NOLINT + + ASSERT_TRUE(arr2.size() == 5); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + ASSERT_TRUE(arr2[3] == 4); + ASSERT_TRUE(arr2[4] == 5); + } +} + TEST(Common, PODArrayInsert) { std::string str = "test_string_abacaba"; diff --git a/src/Common/tests/pod_array.cpp b/src/Common/tests/pod_array.cpp deleted file mode 100644 index 7ebf2670271..00000000000 --- a/src/Common/tests/pod_array.cpp +++ /dev/null @@ -1,455 +0,0 @@ -#include -#include -#include - -#define ASSERT_CHECK(cond, res) \ -do \ -{ \ - if (!(cond)) \ - { \ - std::cerr << __FILE__ << ":" << __LINE__ << ":" \ - << "Assertion " << #cond << " failed.\n"; \ - if ((res)) { (res) = false; } \ - } \ -} \ -while (false) - -static void test1() -{ - using namespace DB; - - static constexpr size_t initial_bytes = 32; - using Array = PODArray, initial_bytes>>; - - bool res = true; - - { - Array arr; - Array arr2; - arr2 = std::move(arr); - } - - { - Array arr; - - arr.push_back(1); - arr.push_back(2); - arr.push_back(3); - - Array arr2; - - arr2 = std::move(arr); - - ASSERT_CHECK((arr2.size() == 3), res); - ASSERT_CHECK((arr2[0] == 1), res); - ASSERT_CHECK((arr2[1] == 2), res); - ASSERT_CHECK((arr2[2] == 3), res); - - arr = std::move(arr2); - - ASSERT_CHECK((arr.size() == 3), res); - ASSERT_CHECK((arr[0] == 1), res); - ASSERT_CHECK((arr[1] == 2), res); - ASSERT_CHECK((arr[2] == 3), res); - } - - { - Array arr; - - arr.push_back(1); - arr.push_back(2); - arr.push_back(3); - arr.push_back(4); - arr.push_back(5); - - Array arr2; - - arr2 = std::move(arr); - - ASSERT_CHECK((arr2.size() == 5), res); - ASSERT_CHECK((arr2[0] == 1), res); - ASSERT_CHECK((arr2[1] == 2), res); - ASSERT_CHECK((arr2[2] == 3), res); - ASSERT_CHECK((arr2[3] == 4), res); - ASSERT_CHECK((arr2[4] == 5), res); - - arr = std::move(arr2); - - ASSERT_CHECK((arr.size() == 5), res); - ASSERT_CHECK((arr[0] == 1), res); - ASSERT_CHECK((arr[1] == 2), res); - ASSERT_CHECK((arr[2] == 3), res); - ASSERT_CHECK((arr[3] == 4), res); - ASSERT_CHECK((arr[4] == 5), res); - } - - { - Array arr; - - arr.push_back(1); - arr.push_back(2); - arr.push_back(3); - - Array arr2; - - arr2.push_back(4); - arr2.push_back(5); - arr2.push_back(6); - arr2.push_back(7); - - arr2 = std::move(arr); - - ASSERT_CHECK((arr2.size() == 3), res); - ASSERT_CHECK((arr2[0] == 1), res); - ASSERT_CHECK((arr2[1] == 2), res); - ASSERT_CHECK((arr2[2] == 3), res); - } - - { - Array arr; - - arr.push_back(1); - arr.push_back(2); - arr.push_back(3); - - Array arr2; - - arr2.push_back(4); - arr2.push_back(5); - arr2.push_back(6); - arr2.push_back(7); - arr2.push_back(8); - - arr = std::move(arr2); - - ASSERT_CHECK((arr.size() == 5), res); - ASSERT_CHECK((arr[0] == 4), res); - ASSERT_CHECK((arr[1] == 5), res); - ASSERT_CHECK((arr[2] == 6), res); - ASSERT_CHECK((arr[3] == 7), res); - ASSERT_CHECK((arr[4] == 8), res); - } - - if (!res) - std::cerr << "Some errors were found in test 1\n"; -} - -static void test2() -{ - using namespace DB; - - static constexpr size_t initial_bytes = 32; - using Array = PODArray, initial_bytes>>; - - bool res = true; - - { - Array arr; - Array arr2; - arr.swap(arr2); - arr2.swap(arr); - } - - { - Array arr; - - Array arr2; - - arr2.push_back(1); - arr2.push_back(2); - arr2.push_back(3); - - arr.swap(arr2); - - ASSERT_CHECK((arr.size() == 3), res); - ASSERT_CHECK((arr[0] == 1), res); - ASSERT_CHECK((arr[1] == 2), res); - ASSERT_CHECK((arr[2] == 3), res); - - ASSERT_CHECK((arr2.empty()), res); - - arr.swap(arr2); - - ASSERT_CHECK((arr.empty()), res); - - ASSERT_CHECK((arr2.size() == 3), res); - ASSERT_CHECK((arr2[0] == 1), res); - ASSERT_CHECK((arr2[1] == 2), res); - ASSERT_CHECK((arr2[2] == 3), res); - } - - { - Array arr; - - Array arr2; - - arr2.push_back(1); - arr2.push_back(2); - arr2.push_back(3); - arr2.push_back(4); - arr2.push_back(5); - - arr.swap(arr2); - - ASSERT_CHECK((arr.size() == 5), res); - ASSERT_CHECK((arr[0] == 1), res); - ASSERT_CHECK((arr[1] == 2), res); - ASSERT_CHECK((arr[2] == 3), res); - ASSERT_CHECK((arr[3] == 4), res); - ASSERT_CHECK((arr[4] == 5), res); - - ASSERT_CHECK((arr2.empty()), res); - - arr.swap(arr2); - - ASSERT_CHECK((arr.empty()), res); - - ASSERT_CHECK((arr2.size() == 5), res); - ASSERT_CHECK((arr2[0] == 1), res); - ASSERT_CHECK((arr2[1] == 2), res); - ASSERT_CHECK((arr2[2] == 3), res); - ASSERT_CHECK((arr2[3] == 4), res); - ASSERT_CHECK((arr2[4] == 5), res); - } - - { - Array arr; - - arr.push_back(1); - arr.push_back(2); - arr.push_back(3); - - Array arr2; - - arr2.push_back(4); - arr2.push_back(5); - arr2.push_back(6); - - arr.swap(arr2); - - ASSERT_CHECK((arr.size() == 3), res); - ASSERT_CHECK((arr[0] == 4), res); - ASSERT_CHECK((arr[1] == 5), res); - ASSERT_CHECK((arr[2] == 6), res); - - ASSERT_CHECK((arr2.size() == 3), res); - ASSERT_CHECK((arr2[0] == 1), res); - ASSERT_CHECK((arr2[1] == 2), res); - ASSERT_CHECK((arr2[2] == 3), res); - - arr.swap(arr2); - - ASSERT_CHECK((arr.size() == 3), res); - ASSERT_CHECK((arr[0] == 1), res); - ASSERT_CHECK((arr[1] == 2), res); - ASSERT_CHECK((arr[2] == 3), res); - - ASSERT_CHECK((arr2.size() == 3), res); - ASSERT_CHECK((arr2[0] == 4), res); - ASSERT_CHECK((arr2[1] == 5), res); - ASSERT_CHECK((arr2[2] == 6), res); - } - - { - Array arr; - - arr.push_back(1); - arr.push_back(2); - - Array arr2; - - arr2.push_back(3); - arr2.push_back(4); - arr2.push_back(5); - - arr.swap(arr2); - - ASSERT_CHECK((arr.size() == 3), res); - ASSERT_CHECK((arr[0] == 3), res); - ASSERT_CHECK((arr[1] == 4), res); - ASSERT_CHECK((arr[2] == 5), res); - - ASSERT_CHECK((arr2.size() == 2), res); - ASSERT_CHECK((arr2[0] == 1), res); - ASSERT_CHECK((arr2[1] == 2), res); - - arr.swap(arr2); - - ASSERT_CHECK((arr.size() == 2), res); - ASSERT_CHECK((arr[0] == 1), res); - ASSERT_CHECK((arr[1] == 2), res); - - ASSERT_CHECK((arr2.size() == 3), res); - ASSERT_CHECK((arr2[0] == 3), res); - ASSERT_CHECK((arr2[1] == 4), res); - ASSERT_CHECK((arr2[2] == 5), res); - } - - { - Array arr; - - arr.push_back(1); - arr.push_back(2); - arr.push_back(3); - - Array arr2; - - arr2.push_back(4); - arr2.push_back(5); - arr2.push_back(6); - arr2.push_back(7); - arr2.push_back(8); - - arr.swap(arr2); - - ASSERT_CHECK((arr.size() == 5), res); - ASSERT_CHECK((arr[0] == 4), res); - ASSERT_CHECK((arr[1] == 5), res); - ASSERT_CHECK((arr[2] == 6), res); - ASSERT_CHECK((arr[3] == 7), res); - ASSERT_CHECK((arr[4] == 8), res); - - ASSERT_CHECK((arr2.size() == 3), res); - ASSERT_CHECK((arr2[0] == 1), res); - ASSERT_CHECK((arr2[1] == 2), res); - ASSERT_CHECK((arr2[2] == 3), res); - - arr.swap(arr2); - - ASSERT_CHECK((arr.size() == 3), res); - ASSERT_CHECK((arr[0] == 1), res); - ASSERT_CHECK((arr[1] == 2), res); - ASSERT_CHECK((arr[2] == 3), res); - - ASSERT_CHECK((arr2.size() == 5), res); - ASSERT_CHECK((arr2[0] == 4), res); - ASSERT_CHECK((arr2[1] == 5), res); - ASSERT_CHECK((arr2[2] == 6), res); - ASSERT_CHECK((arr2[3] == 7), res); - ASSERT_CHECK((arr2[4] == 8), res); - } - - { - Array arr; - - arr.push_back(1); - arr.push_back(2); - arr.push_back(3); - arr.push_back(4); - arr.push_back(5); - - Array arr2; - - arr2.push_back(6); - arr2.push_back(7); - arr2.push_back(8); - arr2.push_back(9); - arr2.push_back(10); - - arr.swap(arr2); - - ASSERT_CHECK((arr.size() == 5), res); - ASSERT_CHECK((arr[0] == 6), res); - ASSERT_CHECK((arr[1] == 7), res); - ASSERT_CHECK((arr[2] == 8), res); - ASSERT_CHECK((arr[3] == 9), res); - ASSERT_CHECK((arr[4] == 10), res); - - ASSERT_CHECK((arr2.size() == 5), res); - ASSERT_CHECK((arr2[0] == 1), res); - ASSERT_CHECK((arr2[1] == 2), res); - ASSERT_CHECK((arr2[2] == 3), res); - ASSERT_CHECK((arr2[3] == 4), res); - ASSERT_CHECK((arr2[4] == 5), res); - - arr.swap(arr2); - - ASSERT_CHECK((arr.size() == 5), res); - ASSERT_CHECK((arr[0] == 1), res); - ASSERT_CHECK((arr[1] == 2), res); - ASSERT_CHECK((arr[2] == 3), res); - ASSERT_CHECK((arr[3] == 4), res); - ASSERT_CHECK((arr[4] == 5), res); - - ASSERT_CHECK((arr2.size() == 5), res); - ASSERT_CHECK((arr2[0] == 6), res); - ASSERT_CHECK((arr2[1] == 7), res); - ASSERT_CHECK((arr2[2] == 8), res); - ASSERT_CHECK((arr2[3] == 9), res); - ASSERT_CHECK((arr2[4] == 10), res); - } - - if (!res) - std::cerr << "Some errors were found in test 2\n"; -} - -static void test3() -{ - using namespace DB; - - static constexpr size_t initial_bytes = 32; - using Array = PODArray, initial_bytes>>; - - bool res = true; - - { - Array arr; - Array arr2{std::move(arr)}; - } - - { - Array arr; - - arr.push_back(1); - arr.push_back(2); - arr.push_back(3); - - Array arr2{std::move(arr)}; - - ASSERT_CHECK((arr.empty()), res); // NOLINT - - ASSERT_CHECK((arr2.size() == 3), res); - ASSERT_CHECK((arr2[0] == 1), res); - ASSERT_CHECK((arr2[1] == 2), res); - ASSERT_CHECK((arr2[2] == 3), res); - } - - { - Array arr; - - arr.push_back(1); - arr.push_back(2); - arr.push_back(3); - arr.push_back(4); - arr.push_back(5); - - Array arr2{std::move(arr)}; - - ASSERT_CHECK((arr.empty()), res); // NOLINT - - ASSERT_CHECK((arr2.size() == 5), res); - ASSERT_CHECK((arr2[0] == 1), res); - ASSERT_CHECK((arr2[1] == 2), res); - ASSERT_CHECK((arr2[2] == 3), res); - ASSERT_CHECK((arr2[3] == 4), res); - ASSERT_CHECK((arr2[4] == 5), res); - } - - if (!res) - std::cerr << "Some errors were found in test 3\n"; -} - -int main() -{ - std::cout << "test 1\n"; - test1(); - std::cout << "test 2\n"; - test2(); - std::cout << "test 3\n"; - test3(); - - return 0; -} diff --git a/src/Common/ya.make b/src/Common/ya.make index d1ff04f8f0a..c6605e436c0 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -47,6 +47,7 @@ SRCS( FileChecker.cpp IPv6ToBinary.cpp IntervalKind.cpp + JSONBuilder.cpp Macros.cpp MemoryStatisticsOS.cpp MemoryTracker.cpp diff --git a/src/Compression/CMakeLists.txt b/src/Compression/CMakeLists.txt index 36cab0b3590..390835f17ae 100644 --- a/src/Compression/CMakeLists.txt +++ b/src/Compression/CMakeLists.txt @@ -1,3 +1,3 @@ -if(ENABLE_TESTS) - add_subdirectory(tests) +if(ENABLE_EXAMPLES) + add_subdirectory(examples) endif() diff --git a/src/Compression/tests/CMakeLists.txt b/src/Compression/examples/CMakeLists.txt similarity index 100% rename from src/Compression/tests/CMakeLists.txt rename to src/Compression/examples/CMakeLists.txt diff --git a/src/Compression/tests/cached_compressed_read_buffer.cpp b/src/Compression/examples/cached_compressed_read_buffer.cpp similarity index 100% rename from src/Compression/tests/cached_compressed_read_buffer.cpp rename to src/Compression/examples/cached_compressed_read_buffer.cpp diff --git a/src/Compression/tests/compressed_buffer.cpp b/src/Compression/examples/compressed_buffer.cpp similarity index 100% rename from src/Compression/tests/compressed_buffer.cpp rename to src/Compression/examples/compressed_buffer.cpp diff --git a/src/Compression/tests/compressed_buffer_fuzzer.cpp b/src/Compression/examples/compressed_buffer_fuzzer.cpp similarity index 100% rename from src/Compression/tests/compressed_buffer_fuzzer.cpp rename to src/Compression/examples/compressed_buffer_fuzzer.cpp diff --git a/src/Core/CMakeLists.txt b/src/Core/CMakeLists.txt index 65172356645..a6176efc7f3 100644 --- a/src/Core/CMakeLists.txt +++ b/src/Core/CMakeLists.txt @@ -1,3 +1,3 @@ -if (ENABLE_TESTS) - add_subdirectory (tests) +if (ENABLE_EXAMPLES) + add_subdirectory(examples) endif () diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 51ea501b949..42a20441a2e 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -435,7 +435,7 @@ class IColumn; M(Bool, allow_experimental_map_type, false, "Allow data type Map", 0) \ M(Bool, allow_experimental_window_functions, false, "Allow experimental window functions", 0) \ M(Bool, use_antlr_parser, false, "Parse incoming queries using ANTLR-generated experimental parser", 0) \ - M(Bool, async_socket_for_remote, false, "Asynchronously read from socket executing remote query", 0) \ + M(Bool, async_socket_for_remote, true, "Asynchronously read from socket executing remote query", 0) \ M(Bool, insert_null_as_default, true, "Insert DEFAULT values instead of NULL in INSERT SELECT (UNION ALL)", 0) \ \ M(Bool, optimize_rewrite_sum_if_to_count_if, true, "Rewrite sumIf() and sum(if()) function countIf() function when logically equivalent", 0) \ diff --git a/src/Core/SortDescription.cpp b/src/Core/SortDescription.cpp index cb7378cf096..314b6624623 100644 --- a/src/Core/SortDescription.cpp +++ b/src/Core/SortDescription.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB { @@ -37,6 +38,22 @@ void dumpSortDescription(const SortDescription & description, const Block & head } } +void SortColumnDescription::explain(JSONBuilder::JSONMap & map, const Block & header) const +{ + if (!column_name.empty()) + map.add("Column", column_name); + else + { + if (column_number < header.columns()) + map.add("Column", header.getByPosition(column_number).name); + + map.add("Position", column_number); + } + + map.add("Ascending", direction > 0); + map.add("With Fill", with_fill); +} + std::string dumpSortDescription(const SortDescription & description) { WriteBufferFromOwnString wb; @@ -44,5 +61,17 @@ std::string dumpSortDescription(const SortDescription & description) return wb.str(); } +JSONBuilder::ItemPtr explainSortDescription(const SortDescription & description, const Block & header) +{ + auto json_array = std::make_unique(); + for (const auto & descr : description) + { + auto json_map = std::make_unique(); + descr.explain(*json_map, header); + json_array->add(std::move(json_map)); + } + + return json_array; } +} diff --git a/src/Core/SortDescription.h b/src/Core/SortDescription.h index 1450393ebd8..41b4e5b6b32 100644 --- a/src/Core/SortDescription.h +++ b/src/Core/SortDescription.h @@ -12,6 +12,15 @@ class Collator; namespace DB { +namespace JSONBuilder +{ + class JSONMap; + class IItem; + using ItemPtr = std::unique_ptr; +} + +class Block; + struct FillColumnDescription { /// All missed values in range [FROM, TO) will be filled @@ -62,16 +71,18 @@ struct SortColumnDescription { return fmt::format("{}:{}:dir {}nulls ", column_name, column_number, direction, nulls_direction); } + + void explain(JSONBuilder::JSONMap & map, const Block & header) const; }; /// Description of the sorting rule for several columns. using SortDescription = std::vector; -class Block; - /// Outputs user-readable description into `out`. void dumpSortDescription(const SortDescription & description, const Block & header, WriteBuffer & out); std::string dumpSortDescription(const SortDescription & description); +JSONBuilder::ItemPtr explainSortDescription(const SortDescription & description, const Block & header); + } diff --git a/src/Core/tests/CMakeLists.txt b/src/Core/examples/CMakeLists.txt similarity index 100% rename from src/Core/tests/CMakeLists.txt rename to src/Core/examples/CMakeLists.txt diff --git a/src/Core/tests/field.cpp b/src/Core/examples/field.cpp similarity index 100% rename from src/Core/tests/field.cpp rename to src/Core/examples/field.cpp diff --git a/src/Core/tests/mysql_protocol.cpp b/src/Core/examples/mysql_protocol.cpp similarity index 100% rename from src/Core/tests/mysql_protocol.cpp rename to src/Core/examples/mysql_protocol.cpp diff --git a/src/Core/tests/names_and_types_fuzzer.cpp b/src/Core/examples/names_and_types_fuzzer.cpp similarity index 100% rename from src/Core/tests/names_and_types_fuzzer.cpp rename to src/Core/examples/names_and_types_fuzzer.cpp diff --git a/src/Core/tests/string_pool.cpp b/src/Core/examples/string_pool.cpp similarity index 100% rename from src/Core/tests/string_pool.cpp rename to src/Core/examples/string_pool.cpp diff --git a/src/Core/tests/string_ref_hash.cpp b/src/Core/examples/string_ref_hash.cpp similarity index 100% rename from src/Core/tests/string_ref_hash.cpp rename to src/Core/examples/string_ref_hash.cpp diff --git a/src/DataStreams/CMakeLists.txt b/src/DataStreams/CMakeLists.txt index 65172356645..a6176efc7f3 100644 --- a/src/DataStreams/CMakeLists.txt +++ b/src/DataStreams/CMakeLists.txt @@ -1,3 +1,3 @@ -if (ENABLE_TESTS) - add_subdirectory (tests) +if (ENABLE_EXAMPLES) + add_subdirectory(examples) endif () diff --git a/src/DataStreams/tests/CMakeLists.txt b/src/DataStreams/examples/CMakeLists.txt similarity index 100% rename from src/DataStreams/tests/CMakeLists.txt rename to src/DataStreams/examples/CMakeLists.txt diff --git a/src/DataTypes/CMakeLists.txt b/src/DataTypes/CMakeLists.txt index 65172356645..a6176efc7f3 100644 --- a/src/DataTypes/CMakeLists.txt +++ b/src/DataTypes/CMakeLists.txt @@ -1,3 +1,3 @@ -if (ENABLE_TESTS) - add_subdirectory (tests) +if (ENABLE_EXAMPLES) + add_subdirectory(examples) endif () diff --git a/src/DataTypes/Serializations/SerializationNumber.cpp b/src/DataTypes/Serializations/SerializationNumber.cpp index b0a91b11716..36cbd6a5ef8 100644 --- a/src/DataTypes/Serializations/SerializationNumber.cpp +++ b/src/DataTypes/Serializations/SerializationNumber.cpp @@ -33,55 +33,11 @@ void SerializationNumber::deserializeText(IColumn & column, ReadBuffer & istr assert_cast &>(column).getData().push_back(x); } -template -static inline void writeDenormalNumber(T x, WriteBuffer & ostr) -{ - if constexpr (std::is_floating_point_v) - { - if (std::signbit(x)) - { - if (isNaN(x)) - writeCString("-nan", ostr); - else - writeCString("-inf", ostr); - } - else - { - if (isNaN(x)) - writeCString("nan", ostr); - else - writeCString("inf", ostr); - } - } - else - { - /// This function is not called for non floating point numbers. - (void)x; - } -} - - template void SerializationNumber::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { auto x = assert_cast &>(column).getData()[row_num]; - bool is_finite = isFinite(x); - - const bool need_quote = (is_integer_v && (sizeof(T) >= 8) && settings.json.quote_64bit_integers) - || (settings.json.quote_denormals && !is_finite); - - if (need_quote) - writeChar('"', ostr); - - if (is_finite) - writeText(x, ostr); - else if (!settings.json.quote_denormals) - writeCString("null", ostr); - else - writeDenormalNumber(x, ostr); - - if (need_quote) - writeChar('"', ostr); + writeJSONNumber(x, ostr, settings); } template diff --git a/src/DataTypes/tests/CMakeLists.txt b/src/DataTypes/examples/CMakeLists.txt similarity index 100% rename from src/DataTypes/tests/CMakeLists.txt rename to src/DataTypes/examples/CMakeLists.txt diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 6d564bc29a3..9b9ea572c3d 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -91,7 +91,7 @@ void DatabaseAtomic::attachTable(const String & name, const StoragePtr & table, not_in_use = cleanupDetachedTables(); auto table_id = table->getStorageID(); assertDetachedTableNotInUse(table_id.uuid); - DatabaseWithDictionaries::attachTableUnlocked(name, table, lock); + DatabaseOrdinary::attachTableUnlocked(name, table, lock); table_name_to_path.emplace(std::make_pair(name, relative_table_path)); } @@ -99,7 +99,7 @@ StoragePtr DatabaseAtomic::detachTable(const String & name) { DetachedTables not_in_use; std::unique_lock lock(mutex); - auto table = DatabaseWithDictionaries::detachTableUnlocked(name, lock); + auto table = DatabaseOrdinary::detachTableUnlocked(name, lock); table_name_to_path.erase(name); detached_tables.emplace(table->getStorageID().uuid, table); not_in_use = cleanupDetachedTables(); @@ -133,9 +133,10 @@ void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_na /// TODO better detection and recovery Poco::File(table_metadata_path).renameTo(table_metadata_path_drop); /// Mark table as dropped - DatabaseWithDictionaries::detachTableUnlocked(table_name, lock); /// Should never throw + DatabaseOrdinary::detachTableUnlocked(table_name, lock); /// Should never throw table_name_to_path.erase(table_name); } + if (table->storesDataOnDisk()) tryRemoveSymlink(table_name); @@ -156,8 +157,6 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_ return; } - if (exchange && dictionary) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot exchange dictionaries"); if (exchange && !supportsRenameat2()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "RENAME EXCHANGE is not supported"); @@ -174,7 +173,7 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_ /// Path can be not set for DDL dictionaries, but it does not matter for StorageDictionary. if (it != db.table_name_to_path.end()) table_data_path_saved = it->second; - assert(!table_data_path_saved.empty() || db.dictionaries.find(table_name_) != db.dictionaries.end()); + assert(!table_data_path_saved.empty()); db.tables.erase(table_name_); db.table_name_to_path.erase(table_name_); if (has_symlink) @@ -222,21 +221,21 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_ db_lock = std::unique_lock{mutex}; } - bool is_dictionary = dictionaries.find(table_name) != dictionaries.end(); - if (exchange && other_db.dictionaries.find(to_table_name) != other_db.dictionaries.end()) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot exchange dictionaries"); - - if (dictionary != is_dictionary) - throw Exception(ErrorCodes::INCORRECT_QUERY, - "Use RENAME DICTIONARY for dictionaries and RENAME TABLE for tables."); - - if (is_dictionary && !inside_database) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot move dictionary to other database"); - if (!exchange) other_db.checkMetadataFilenameAvailabilityUnlocked(to_table_name, inside_database ? db_lock : other_db_lock); StoragePtr table = getTableUnlocked(table_name, db_lock); + + if (table->isDictionary() && !dictionary) + { + if (exchange) + throw Exception(ErrorCodes::INCORRECT_QUERY, + "Use EXCHANGE DICTIONARIES for dictionaries and EXCHANGE TABLES for tables."); + else + throw Exception(ErrorCodes::INCORRECT_QUERY, + "Use RENAME DICTIONARY for dictionaries and RENAME TABLE for tables."); + } + table->checkTableCanBeRenamed(); assert_can_move_mat_view(table); StoragePtr other_table; @@ -281,12 +280,6 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_ attach(other_db, to_table_name, table_data_path, table); if (exchange) attach(*this, table_name, other_table_data_path, other_table); - - if (is_dictionary) - { - auto new_table_id = StorageID(other_db.database_name, to_table_name, old_table_id.uuid); - renameDictionaryInMemoryUnlocked(old_table_id, new_table_id); - } } void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const StoragePtr & table, @@ -528,14 +521,6 @@ void DatabaseAtomic::renameDatabase(const String & new_name) table.second->renameInMemory(table_id); } - for (auto & dict : dictionaries) - { - auto old_name = StorageID(dict.second.create_query); - auto name = old_name; - name.database_name = database_name; - renameDictionaryInMemoryUnlocked(old_name, name); - } - path_to_metadata_symlink = getContext()->getPath() + "metadata/" + new_name_escaped; old_path_to_table_symlinks = path_to_table_symlinks; path_to_table_symlinks = getContext()->getPath() + "data/" + new_name_escaped + "/"; @@ -545,32 +530,6 @@ void DatabaseAtomic::renameDatabase(const String & new_name) tryCreateMetadataSymlink(); } -void DatabaseAtomic::renameDictionaryInMemoryUnlocked(const StorageID & old_name, const StorageID & new_name) -{ - auto it = dictionaries.find(old_name.table_name); - assert(it != dictionaries.end()); - assert(it->second.config->getString("dictionary.uuid") == toString(old_name.uuid)); - assert(old_name.uuid == new_name.uuid); - it->second.config->setString("dictionary.database", new_name.database_name); - it->second.config->setString("dictionary.name", new_name.table_name); - auto & create = it->second.create_query->as(); - create.database = new_name.database_name; - create.table = new_name.table_name; - assert(create.uuid == new_name.uuid); - - if (old_name.table_name != new_name.table_name) - { - auto attach_info = std::move(it->second); - dictionaries.erase(it); - dictionaries.emplace(new_name.table_name, std::move(attach_info)); - } - - auto result = external_loader.getLoadResult(toString(old_name.uuid)); - if (!result.object) - return; - const auto & dict = dynamic_cast(*result.object); - dict.updateDictionaryName(new_name); -} void DatabaseAtomic::waitDetachedTableNotInUse(const UUID & uuid) { /// Table is in use while its shared_ptr counter is greater than 1. diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h index 695d22360ca..a6acd10b656 100644 --- a/src/Databases/DatabaseAtomic.h +++ b/src/Databases/DatabaseAtomic.h @@ -72,8 +72,6 @@ protected: void tryCreateMetadataSymlink(); - void renameDictionaryInMemoryUnlocked(const StorageID & old_name, const StorageID & new_name); - //TODO store path in DatabaseWithOwnTables::tables using NameToPathMap = std::unordered_map; NameToPathMap table_name_to_path; diff --git a/src/Databases/DatabaseDictionary.cpp b/src/Databases/DatabaseDictionary.cpp index c00201145eb..40034015f27 100644 --- a/src/Databases/DatabaseDictionary.cpp +++ b/src/Databases/DatabaseDictionary.cpp @@ -21,18 +21,20 @@ namespace ErrorCodes namespace { - StoragePtr createStorageDictionary(const String & database_name, const ExternalLoader::LoadResult & load_result) + StoragePtr createStorageDictionary(const String & database_name, const ExternalLoader::LoadResult & load_result, ContextPtr context) { try { if (!load_result.config) return nullptr; + DictionaryStructure dictionary_structure = ExternalDictionariesLoader::getDictionaryStructure(*load_result.config); return StorageDictionary::create( StorageID(database_name, load_result.name), load_result.name, dictionary_structure, - StorageDictionary::Location::DictionaryDatabase); + StorageDictionary::Location::DictionaryDatabase, + context); } catch (Exception & e) { @@ -57,7 +59,7 @@ Tables DatabaseDictionary::listTables(const FilterByNameFunction & filter_by_nam String db_name = getDatabaseName(); for (auto & load_result : load_results) { - auto storage = createStorageDictionary(db_name, load_result); + auto storage = createStorageDictionary(db_name, load_result, getContext()); if (storage) tables.emplace(storage->getStorageID().table_name, storage); } @@ -72,7 +74,7 @@ bool DatabaseDictionary::isTableExist(const String & table_name, ContextPtr) con StoragePtr DatabaseDictionary::tryGetTable(const String & table_name, ContextPtr) const { auto load_result = getContext()->getExternalDictionariesLoader().getLoadResult(table_name); - return createStorageDictionary(getDatabaseName(), load_result); + return createStorageDictionary(getDatabaseName(), load_result, getContext()); } DatabaseTablesIteratorPtr DatabaseDictionary::getTablesIterator(ContextPtr, const FilterByNameFunction & filter_by_table_name) diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 14ad1c7e4c5..2ad961ccce4 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -37,7 +37,6 @@ namespace ErrorCodes extern const int INCORRECT_FILE_NAME; extern const int SYNTAX_ERROR; extern const int TABLE_ALREADY_EXISTS; - extern const int DICTIONARY_ALREADY_EXISTS; extern const int EMPTY_LIST_OF_COLUMNS_PASSED; } @@ -63,14 +62,21 @@ std::pair createTableFromAST( storage->renameInMemory(ast_create_query); return {ast_create_query.table, storage}; } - /// We do not directly use `InterpreterCreateQuery::execute`, because - /// - the database has not been loaded yet; - /// - the code is simpler, since the query is already brought to a suitable form. - if (!ast_create_query.columns_list || !ast_create_query.columns_list->columns) - throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); - ColumnsDescription columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true); - ConstraintsDescription constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints); + ColumnsDescription columns; + ConstraintsDescription constraints; + + if (!ast_create_query.is_dictionary) + { + /// We do not directly use `InterpreterCreateQuery::execute`, because + /// - the database has not been loaded yet; + /// - the code is simpler, since the query is already brought to a suitable form. + if (!ast_create_query.columns_list || !ast_create_query.columns_list->columns) + throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); + + columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true); + constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints); + } return { @@ -220,10 +226,6 @@ void DatabaseOnDisk::createTable( /// A race condition would be possible if a table with the same name is simultaneously created using CREATE and using ATTACH. /// But there is protection from it - see using DDLGuard in InterpreterCreateQuery. - if (isDictionaryExist(table_name)) - throw Exception( - ErrorCodes::DICTIONARY_ALREADY_EXISTS, "Dictionary {}.{} already exists", backQuote(getDatabaseName()), backQuote(table_name)); - if (isTableExist(table_name, getContext())) throw Exception( ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists", backQuote(getDatabaseName()), backQuote(table_name)); diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 840be0e006a..cea31d88d57 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -43,13 +43,15 @@ namespace const String & metadata_path, bool has_force_restore_data_flag) { - assert(!query.is_dictionary); try { - String table_name; - StoragePtr table; - std::tie(table_name, table) - = createTableFromAST(query, database_name, database.getTableDataPath(query), context, has_force_restore_data_flag); + auto [table_name, table] = createTableFromAST( + query, + database_name, + database.getTableDataPath(query), + context, + has_force_restore_data_flag); + database.attachTable(table_name, table, database.getTableDataPath(query)); } catch (Exception & e) @@ -61,28 +63,6 @@ namespace } } - - void tryAttachDictionary(const ASTPtr & query, DatabaseOrdinary & database, const String & metadata_path, ContextPtr context) - { - auto & create_query = query->as(); - assert(create_query.is_dictionary); - try - { - Poco::File meta_file(metadata_path); - auto config = getDictionaryConfigurationFromAST(create_query, context, database.getDatabaseName()); - time_t modification_time = meta_file.getLastModified().epochTime(); - database.attachDictionary(create_query.table, DictionaryAttachInfo{query, config, modification_time}); - } - catch (Exception & e) - { - e.addMessage( - "Cannot attach dictionary " + backQuote(database.getDatabaseName()) + "." + backQuote(create_query.table) - + " from metadata file " + metadata_path + " from query " + serializeAST(*query)); - throw; - } - } - - void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, AtomicStopwatch & watch) { if (processed % PRINT_MESSAGE_EACH_N_OBJECTS == 0 || watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS)) @@ -101,7 +81,7 @@ DatabaseOrdinary::DatabaseOrdinary(const String & name_, const String & metadata DatabaseOrdinary::DatabaseOrdinary( const String & name_, const String & metadata_path_, const String & data_path_, const String & logger, ContextPtr context_) - : DatabaseWithDictionaries(name_, metadata_path_, data_path_, logger, context_) + : DatabaseOnDisk(name_, metadata_path_, data_path_, logger, context_) { } @@ -117,7 +97,7 @@ void DatabaseOrdinary::loadStoredObjects(ContextPtr local_context, bool has_forc size_t total_dictionaries = 0; - auto process_metadata = [context_weak = ContextWeakPtr(local_context), &file_names, &total_dictionaries, &file_names_mutex, this]( + auto process_metadata = [&file_names, &total_dictionaries, &file_names_mutex, this]( const String & file_name) { fs::path path(getMetadataPath()); @@ -164,7 +144,6 @@ void DatabaseOrdinary::loadStoredObjects(ContextPtr local_context, bool has_forc AtomicStopwatch watch; std::atomic tables_processed{0}; - std::atomic dictionaries_processed{0}; ThreadPool pool; @@ -176,23 +155,12 @@ void DatabaseOrdinary::loadStoredObjects(ContextPtr local_context, bool has_forc /// loading of its config only, it doesn't involve loading the dictionary itself. /// Attach dictionaries. - for (const auto & [name, query] : file_names) - { - auto create_query = query->as(); - if (create_query.is_dictionary) - { - tryAttachDictionary(query, *this, getMetadataPath() + name, local_context); - - /// Messages, so that it's not boring to wait for the server to load for a long time. - logAboutProgress(log, ++dictionaries_processed, total_dictionaries, watch); - } - } - - /// Attach tables. for (const auto & name_with_query : file_names) { const auto & create_query = name_with_query.second->as(); - if (!create_query.is_dictionary) + + if (create_query.is_dictionary) + { pool.scheduleOrThrowOnError([&]() { tryAttachTable( @@ -206,6 +174,32 @@ void DatabaseOrdinary::loadStoredObjects(ContextPtr local_context, bool has_forc /// Messages, so that it's not boring to wait for the server to load for a long time. logAboutProgress(log, ++tables_processed, total_tables, watch); }); + } + } + + pool.wait(); + + /// Attach tables. + for (const auto & name_with_query : file_names) + { + const auto & create_query = name_with_query.second->as(); + + if (!create_query.is_dictionary) + { + pool.scheduleOrThrowOnError([&]() + { + tryAttachTable( + local_context, + create_query, + *this, + database_name, + getMetadataPath() + name_with_query.first, + has_force_restore_data_flag); + + /// Messages, so that it's not boring to wait for the server to load for a long time. + logAboutProgress(log, ++tables_processed, total_tables, watch); + }); + } } pool.wait(); diff --git a/src/Databases/DatabaseOrdinary.h b/src/Databases/DatabaseOrdinary.h index 4cf58cef9f0..8a32bb10991 100644 --- a/src/Databases/DatabaseOrdinary.h +++ b/src/Databases/DatabaseOrdinary.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include @@ -11,7 +11,7 @@ namespace DB * It stores tables list in filesystem using list of .sql files, * that contain declaration of table represented by SQL ATTACH TABLE query. */ -class DatabaseOrdinary : public DatabaseWithDictionaries +class DatabaseOrdinary : public DatabaseOnDisk { public: DatabaseOrdinary(const String & name_, const String & metadata_path_, ContextPtr context); diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 46d67e275ba..b7214917ce8 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -511,9 +511,10 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep executeQuery(query, query_context, true); } - size_t dropped_dicts = 0; size_t moved_tables = 0; std::vector dropped_tables; + size_t dropped_dictionaries = 0; + for (const auto & table_name : tables_to_detach) { DDLGuardPtr table_guard = DatabaseCatalog::instance().getDDLGuard(db_name, table_name); @@ -521,17 +522,13 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database was renamed, will retry"); auto table = tryGetTable(table_name, getContext()); - if (isDictionaryExist(table_name)) - { - /// We can safely drop any dictionaries because they do not store data - LOG_DEBUG(log, "Will DROP DICTIONARY {}", backQuoteIfNeed(table_name)); - DatabaseAtomic::removeDictionary(getContext(), table_name); - ++dropped_dicts; - } - else if (!table->storesDataOnDisk()) + + if (!table->storesDataOnDisk()) { LOG_DEBUG(log, "Will DROP TABLE {}, because it does not store data on disk and can be safely dropped", backQuoteIfNeed(table_name)); dropped_tables.push_back(tryGetTableUUID(table_name)); + dropped_dictionaries += table->isDictionary(); + table->shutdown(); DatabaseAtomic::dropTable(getContext(), table_name, true); } @@ -550,7 +547,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep if (!tables_to_detach.empty()) LOG_WARNING(log, "Cleaned {} outdated objects: dropped {} dictionaries and {} tables, moved {} tables", - tables_to_detach.size(), dropped_dicts, dropped_tables.size(), moved_tables); + tables_to_detach.size(), dropped_dictionaries, dropped_tables.size() - dropped_dictionaries, moved_tables); /// Now database is cleared from outdated tables, let's rename ReplicatedMergeTree tables to actual names for (const auto & old_to_new : replicated_tables_to_rename) @@ -766,33 +763,6 @@ void DatabaseReplicated::commitAlterTable(const StorageID & table_id, DatabaseAtomic::commitAlterTable(table_id, table_metadata_tmp_path, table_metadata_path, statement, query_context); } -void DatabaseReplicated::createDictionary(ContextPtr local_context, - const String & dictionary_name, - const ASTPtr & query) -{ - auto txn = local_context->getZooKeeperMetadataTransaction(); - assert(!ddl_worker->isCurrentlyActive() || txn); - if (txn && txn->isInitialQuery()) - { - String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(dictionary_name); - String statement = getObjectDefinitionFromCreateQuery(query->clone()); - txn->addOp(zkutil::makeCreateRequest(metadata_zk_path, statement, zkutil::CreateMode::Persistent)); - } - DatabaseAtomic::createDictionary(local_context, dictionary_name, query); -} - -void DatabaseReplicated::removeDictionary(ContextPtr local_context, const String & dictionary_name) -{ - auto txn = local_context->getZooKeeperMetadataTransaction(); - assert(!ddl_worker->isCurrentlyActive() || txn); - if (txn && txn->isInitialQuery()) - { - String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(dictionary_name); - txn->addOp(zkutil::makeRemoveRequest(metadata_zk_path, -1)); - } - DatabaseAtomic::removeDictionary(local_context, dictionary_name); -} - void DatabaseReplicated::detachTablePermanently(ContextPtr local_context, const String & table_name) { auto txn = local_context->getZooKeeperMetadataTransaction(); diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 5220535f095..b930d27c19b 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -40,10 +40,6 @@ public: void commitAlterTable(const StorageID & table_id, const String & table_metadata_tmp_path, const String & table_metadata_path, const String & statement, ContextPtr query_context) override; - void createDictionary(ContextPtr context, - const String & dictionary_name, - const ASTPtr & query) override; - void removeDictionary(ContextPtr context, const String & dictionary_name) override; void detachTablePermanently(ContextPtr context, const String & table_name) override; void removeDetachedPermanentlyFlag(ContextPtr context, const String & table_name, const String & table_metadata_path, bool attach) const override; diff --git a/src/Databases/DatabaseWithDictionaries.cpp b/src/Databases/DatabaseWithDictionaries.cpp deleted file mode 100644 index c97417e292c..00000000000 --- a/src/Databases/DatabaseWithDictionaries.cpp +++ /dev/null @@ -1,381 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace CurrentStatusInfo -{ - extern const Status DictionaryStatus; -} - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY; - extern const int TABLE_ALREADY_EXISTS; - extern const int UNKNOWN_DICTIONARY; - extern const int DICTIONARY_ALREADY_EXISTS; - extern const int FILE_DOESNT_EXIST; -} - - -void DatabaseWithDictionaries::attachDictionary(const String & dictionary_name, const DictionaryAttachInfo & attach_info) -{ - auto dict_id = StorageID(attach_info.create_query); - String internal_name = dict_id.getInternalDictionaryName(); - assert(attach_info.create_query->as().table == dictionary_name); - assert(!dict_id.database_name.empty()); - { - std::unique_lock lock(mutex); - auto [it, inserted] = dictionaries.emplace(dictionary_name, attach_info); - if (!inserted) - throw Exception(ErrorCodes::DICTIONARY_ALREADY_EXISTS, - "Dictionary {} already exists.", dict_id.getNameForLogs()); - - /// Attach the dictionary as table too. - try - { - /// TODO Make StorageDictionary an owner of IDictionary objects. - /// All DDL operations with dictionaries will work with StorageDictionary table, - /// and StorageDictionary will be responsible for loading of DDL dictionaries. - /// ExternalLoaderDatabaseConfigRepository and other hacks related to ExternalLoader - /// will not be longer required. - attachTableUnlocked( - dictionary_name, - StorageDictionary::create( - dict_id, - internal_name, - ExternalDictionariesLoader::getDictionaryStructure(*attach_info.config), - StorageDictionary::Location::SameDatabaseAndNameAsDictionary), - lock); - } - catch (...) - { - dictionaries.erase(it); - throw; - } - } - - CurrentStatusInfo::set(CurrentStatusInfo::DictionaryStatus, internal_name, static_cast(ExternalLoaderStatus::NOT_LOADED)); - - /// We want ExternalLoader::reloadConfig() to find out that the dictionary's config - /// has been added and in case `dictionaries_lazy_load == false` to load the dictionary. - reloadDictionaryConfig(internal_name); -} - -void DatabaseWithDictionaries::detachDictionary(const String & dictionary_name) -{ - DictionaryAttachInfo attach_info; - detachDictionaryImpl(dictionary_name, attach_info); -} - -void DatabaseWithDictionaries::detachDictionaryImpl(const String & dictionary_name, DictionaryAttachInfo & attach_info) -{ - auto dict_id = StorageID::createEmpty(); - String internal_name; - - { - std::unique_lock lock(mutex); - auto it = dictionaries.find(dictionary_name); - if (it == dictionaries.end()) - throw Exception(ErrorCodes::UNKNOWN_DICTIONARY, - "Dictionary {}.{} doesn't exist.", database_name, dictionary_name); - dict_id = StorageID(it->second.create_query); - internal_name = dict_id.getInternalDictionaryName(); - assert(dict_id.table_name == dictionary_name); - assert(!dict_id.database_name.empty()); - - attach_info = std::move(it->second); - dictionaries.erase(it); - - /// Detach the dictionary as table too. - try - { - if (!dict_id.hasUUID()) - detachTableUnlocked(dictionary_name, lock); - } - catch (...) - { - dictionaries.emplace(dictionary_name, std::move(attach_info)); - throw; - } - } - - CurrentStatusInfo::unset(CurrentStatusInfo::DictionaryStatus, internal_name); - - /// We want ExternalLoader::reloadConfig() to find out that the dictionary's config - /// has been removed and to unload the dictionary. - reloadDictionaryConfig(internal_name); - - if (dict_id.hasUUID()) - detachTable(dictionary_name); -} - -void DatabaseWithDictionaries::createDictionary(ContextPtr local_context, const String & dictionary_name, const ASTPtr & query) -{ - const auto & settings = local_context->getSettingsRef(); - - /** The code is based on the assumption that all threads share the same order of operations: - * - create the .sql.tmp file; - * - add the dictionary to ExternalDictionariesLoader; - * - load the dictionary in case dictionaries_lazy_load == false; - * - attach the dictionary; - * - rename .sql.tmp to .sql. - */ - - auto dict_id = StorageID(query); - assert(query->as().table == dictionary_name); - assert(!dict_id.database_name.empty()); - - /// A race condition would be possible if a dictionary with the same name is simultaneously created using CREATE and using ATTACH. - /// But there is protection from it - see using DDLGuard in InterpreterCreateQuery. - if (isDictionaryExist(dictionary_name)) - throw Exception(ErrorCodes::DICTIONARY_ALREADY_EXISTS, "Dictionary {} already exists.", dict_id.getFullTableName()); - - /// A dictionary with the same full name could be defined in *.xml config files. - if (external_loader.getCurrentStatus(dict_id.getFullNameNotQuoted()) != ExternalLoader::Status::NOT_EXIST) - throw Exception(ErrorCodes::DICTIONARY_ALREADY_EXISTS, - "Dictionary {} already exists.", dict_id.getFullNameNotQuoted()); - - if (isTableExist(dictionary_name, getContext())) - throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {} already exists.", dict_id.getFullTableName()); - - String dictionary_metadata_path = getObjectMetadataPath(dictionary_name); - String dictionary_metadata_tmp_path = dictionary_metadata_path + ".tmp"; - String statement = getObjectDefinitionFromCreateQuery(query); - - { - /// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown. - WriteBufferFromFile out(dictionary_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL); - writeString(statement, out); - out.next(); - if (settings.fsync_metadata) - out.sync(); - out.close(); - } - - bool succeeded = false; - bool uuid_locked = false; - SCOPE_EXIT({ - if (!succeeded) - { - if (uuid_locked) - DatabaseCatalog::instance().removeUUIDMappingFinally(dict_id.uuid); - Poco::File(dictionary_metadata_tmp_path).remove(); - } - }); - - if (dict_id.uuid != UUIDHelpers::Nil) - { - DatabaseCatalog::instance().addUUIDMapping(dict_id.uuid); - uuid_locked = true; - } - - /// Add a temporary repository containing the dictionary. - /// We need this temp repository to try loading the dictionary before actually attaching it to the database. - auto temp_repository = external_loader.addConfigRepository(std::make_unique( - getDatabaseName(), dictionary_metadata_tmp_path, getDictionaryConfigurationFromAST(query->as(), local_context))); - - bool lazy_load = local_context->getConfigRef().getBool("dictionaries_lazy_load", true); - if (!lazy_load) - { - /// load() is called here to force loading the dictionary, wait until the loading is finished, - /// and throw an exception if the loading is failed. - external_loader.load(dict_id.getInternalDictionaryName()); - } - - auto config = getDictionaryConfigurationFromAST(query->as(), local_context); - attachDictionary(dictionary_name, DictionaryAttachInfo{query, config, time(nullptr)}); - SCOPE_EXIT({ - if (!succeeded) - detachDictionary(dictionary_name); - }); - - auto txn = local_context->getZooKeeperMetadataTransaction(); - if (txn && !local_context->isInternalSubquery()) - txn->commit(); /// Commit point (a sort of) for Replicated database - - /// If it was ATTACH query and file with dictionary metadata already exist - /// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one. - Poco::File(dictionary_metadata_tmp_path).renameTo(dictionary_metadata_path); - - /// ExternalDictionariesLoader doesn't know we renamed the metadata path. - /// That's why we have to call ExternalLoader::reloadConfig() here. - reloadDictionaryConfig(dict_id.getInternalDictionaryName()); - - /// Everything's ok. - succeeded = true; -} - -void DatabaseWithDictionaries::removeDictionary(ContextPtr local_context, const String & dictionary_name) -{ - DictionaryAttachInfo attach_info; - detachDictionaryImpl(dictionary_name, attach_info); - - try - { - String dictionary_metadata_path = getObjectMetadataPath(dictionary_name); - - auto txn = local_context->getZooKeeperMetadataTransaction(); - if (txn && !local_context->isInternalSubquery()) - txn->commit(); /// Commit point (a sort of) for Replicated database - - Poco::File(dictionary_metadata_path).remove(); - CurrentStatusInfo::unset(CurrentStatusInfo::DictionaryStatus, - StorageID(attach_info.create_query).getInternalDictionaryName()); - } - catch (...) - { - /// If remove was not possible for some reason - attachDictionary(dictionary_name, attach_info); - throw; - } - - UUID dict_uuid = attach_info.create_query->as()->uuid; - if (dict_uuid != UUIDHelpers::Nil) - DatabaseCatalog::instance().removeUUIDMappingFinally(dict_uuid); -} - -DatabaseDictionariesIteratorPtr DatabaseWithDictionaries::getDictionariesIterator(const FilterByNameFunction & filter_by_dictionary_name) -{ - std::lock_guard lock(mutex); - DictionariesWithID filtered_dictionaries; - for (const auto & dictionary : dictionaries) - { - if (filter_by_dictionary_name && !filter_by_dictionary_name(dictionary.first)) - continue; - filtered_dictionaries.emplace_back(); - filtered_dictionaries.back().first = dictionary.first; - filtered_dictionaries.back().second = dictionary.second.create_query->as().uuid; - } - return std::make_unique(std::move(filtered_dictionaries), database_name); -} - -bool DatabaseWithDictionaries::isDictionaryExist(const String & dictionary_name) const -{ - std::lock_guard lock(mutex); - return dictionaries.find(dictionary_name) != dictionaries.end(); -} - -ASTPtr DatabaseWithDictionaries::getCreateDictionaryQueryImpl( - const String & dictionary_name, - bool throw_on_error) const -{ - { - /// Try to get create query ifg for an attached dictionary. - std::lock_guard lock{mutex}; - auto it = dictionaries.find(dictionary_name); - if (it != dictionaries.end()) - { - ASTPtr ast = it->second.create_query->clone(); - auto & create_query = ast->as(); - create_query.attach = false; - create_query.database = database_name; - return ast; - } - } - - /// Try to get create query for non-attached dictionary. - ASTPtr ast; - try - { - auto dictionary_metadata_path = getObjectMetadataPath(dictionary_name); - ast = getCreateQueryFromMetadata(dictionary_metadata_path, throw_on_error); - } - catch (const Exception & e) - { - if (throw_on_error && (e.code() != ErrorCodes::FILE_DOESNT_EXIST)) - throw; - } - - if (ast) - { - const auto * create_query = ast->as(); - if (create_query && create_query->is_dictionary) - return ast; - } - if (throw_on_error) - throw Exception{"Dictionary " + backQuote(dictionary_name) + " doesn't exist", - ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY}; - return nullptr; -} - -Poco::AutoPtr DatabaseWithDictionaries::getDictionaryConfiguration(const String & dictionary_name) const -{ - std::lock_guard lock(mutex); - auto it = dictionaries.find(dictionary_name); - if (it != dictionaries.end()) - return it->second.config; - throw Exception("Dictionary " + backQuote(dictionary_name) + " doesn't exist", ErrorCodes::UNKNOWN_DICTIONARY); -} - -time_t DatabaseWithDictionaries::getObjectMetadataModificationTime(const String & object_name) const -{ - { - std::lock_guard lock(mutex); - auto it = dictionaries.find(object_name); - if (it != dictionaries.end()) - return it->second.modification_time; - } - return DatabaseOnDisk::getObjectMetadataModificationTime(object_name); -} - - -bool DatabaseWithDictionaries::empty() const -{ - std::lock_guard lock{mutex}; - return tables.empty() && dictionaries.empty(); -} - -void DatabaseWithDictionaries::reloadDictionaryConfig(const String & full_name) -{ - /// Ensure that this database is attached to ExternalLoader as a config repository. - if (!database_as_config_repo_for_external_loader.load()) - { - auto repository = std::make_unique(*this, getContext()); - auto remove_repository_callback = external_loader.addConfigRepository(std::move(repository)); - database_as_config_repo_for_external_loader = boost::make_shared(std::move(remove_repository_callback)); - } - - external_loader.reloadConfig(getDatabaseName(), full_name); -} - - -void DatabaseWithDictionaries::shutdown() -{ - { - std::lock_guard lock(mutex); - dictionaries.clear(); - } - - /// Invoke removing the database from ExternalLoader. - database_as_config_repo_for_external_loader = nullptr; - - DatabaseOnDisk::shutdown(); -} - - -DatabaseWithDictionaries::DatabaseWithDictionaries( - const String & name, const String & metadata_path_, const String & data_path_, const String & logger, ContextPtr context_) - : DatabaseOnDisk(name, metadata_path_, data_path_, logger, context_) - , external_loader(context_->getExternalDictionariesLoader()) -{ -} - -DatabaseWithDictionaries::~DatabaseWithDictionaries() = default; - -} diff --git a/src/Databases/DatabaseWithDictionaries.h b/src/Databases/DatabaseWithDictionaries.h deleted file mode 100644 index d10908c7c06..00000000000 --- a/src/Databases/DatabaseWithDictionaries.h +++ /dev/null @@ -1,55 +0,0 @@ -#pragma once -#include -#include -#include - -namespace DB -{ - -class Context; -class ExternalDictionariesLoader; - - -class DatabaseWithDictionaries : public DatabaseOnDisk -{ -public: - void attachDictionary(const String & dictionary_name, const DictionaryAttachInfo & attach_info) override; - - void detachDictionary(const String & dictionary_name) override; - - void createDictionary(ContextPtr context, - const String & dictionary_name, - const ASTPtr & query) override; - - void removeDictionary(ContextPtr context, const String & dictionary_name) override; - - bool isDictionaryExist(const String & dictionary_name) const override; - - DatabaseDictionariesIteratorPtr getDictionariesIterator(const FilterByNameFunction & filter_by_dictionary_name) override; - - Poco::AutoPtr getDictionaryConfiguration(const String & /*name*/) const override; - - time_t getObjectMetadataModificationTime(const String & object_name) const override; - - bool empty() const override; - - void shutdown() override; - - ~DatabaseWithDictionaries() override; - -protected: - DatabaseWithDictionaries(const String & name, const String & metadata_path_, const String & data_path_, const String & logger, ContextPtr context); - - ASTPtr getCreateDictionaryQueryImpl(const String & dictionary_name, bool throw_on_error) const override; - - std::unordered_map dictionaries; - const ExternalDictionariesLoader & external_loader; - -private: - void detachDictionaryImpl(const String & dictionary_name, DictionaryAttachInfo & attach_info); - void reloadDictionaryConfig(const String & full_name); - - boost::atomic_shared_ptr database_as_config_repo_for_external_loader; -}; - -} diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 8c356b88460..e4b6e042f56 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -29,7 +29,6 @@ namespace ErrorCodes { extern const int NOT_IMPLEMENTED; extern const int CANNOT_GET_CREATE_TABLE_QUERY; - extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY; } class IDatabaseTablesIterator @@ -95,38 +94,7 @@ public: const StoragePtr & table() const override { return it->second; } }; -/// Copies list of dictionaries and iterates through such snapshot. -class DatabaseDictionariesSnapshotIterator -{ -private: - DictionariesWithID dictionaries; - DictionariesWithID::iterator it; - String database_name; - -public: - DatabaseDictionariesSnapshotIterator() = default; - DatabaseDictionariesSnapshotIterator(DictionariesWithID & dictionaries_, const String & database_name_) - : dictionaries(dictionaries_), it(dictionaries.begin()), database_name(database_name_) - { - } - DatabaseDictionariesSnapshotIterator(DictionariesWithID && dictionaries_, const String & database_name_) - : dictionaries(dictionaries_), it(dictionaries.begin()), database_name(database_name_) - { - } - - void next() { ++it; } - - bool isValid() const { return !dictionaries.empty() && it != dictionaries.end(); } - - const String & name() const { return it->first; } - - const UUID & uuid() const { return it->second; } - - const String & databaseName() const { assert(!database_name.empty()); return database_name; } -}; - using DatabaseTablesIteratorPtr = std::unique_ptr; -using DatabaseDictionariesIteratorPtr = std::unique_ptr; /** Database engine. @@ -158,12 +126,6 @@ public: /// Check the existence of the table. virtual bool isTableExist(const String & name, ContextPtr context) const = 0; - /// Check the existence of the dictionary - virtual bool isDictionaryExist(const String & /*name*/) const - { - return false; - } - /// Get the table for work. Return nullptr if there is no table. virtual StoragePtr tryGetTable(const String & name, ContextPtr context) const = 0; @@ -175,12 +137,6 @@ public: /// It is possible to have "hidden" tables that are not visible when passing through, but are visible if you get them by name using the functions above. virtual DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name = {}) = 0; - /// Get an iterator to pass through all the dictionaries. - virtual DatabaseDictionariesIteratorPtr getDictionariesIterator([[maybe_unused]] const FilterByNameFunction & filter_by_dictionary_name = {}) - { - return std::make_unique(); - } - /// Is the database empty. virtual bool empty() const = 0; @@ -194,15 +150,6 @@ public: throw Exception("There is no CREATE TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED); } - /// Add the dictionary to the database. Record its presence in the metadata. - virtual void createDictionary( - ContextPtr /*context*/, - const String & /*dictionary_name*/, - const ASTPtr & /*query*/) - { - throw Exception("There is no CREATE DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED); - } - /// Delete the table from the database, drop table and delete the metadata. virtual void dropTable( ContextPtr /*context*/, @@ -212,14 +159,6 @@ public: throw Exception("There is no DROP TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED); } - /// Delete the dictionary from the database. Delete the metadata. - virtual void removeDictionary( - ContextPtr /*context*/, - const String & /*dictionary_name*/) - { - throw Exception("There is no DROP DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED); - } - /// Add a table to the database, but do not add it to the metadata. The database may not support this method. /// /// Note: ATTACH TABLE statement actually uses createTable method. @@ -228,25 +167,12 @@ public: throw Exception("There is no ATTACH TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED); } - /// Add dictionary to the database, but do not add it to the metadata. The database may not support this method. - /// If dictionaries_lazy_load is false it also starts loading the dictionary asynchronously. - virtual void attachDictionary(const String & /* dictionary_name */, const DictionaryAttachInfo & /* attach_info */) - { - throw Exception("There is no ATTACH DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED); - } - /// Forget about the table without deleting it, and return it. The database may not support this method. virtual StoragePtr detachTable(const String & /*name*/) { throw Exception("There is no DETACH TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED); } - /// Forget about the dictionary without deleting it. The database may not support this method. - virtual void detachDictionary(const String & /*name*/) - { - throw Exception("There is no DETACH DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED); - } - /// Forget about the table without deleting it's data, but rename metadata file to prevent reloading it /// with next restart. The database may not support this method. virtual void detachTablePermanently(ContextPtr /*context*/, const String & /*name*/) @@ -295,22 +221,6 @@ public: return getCreateTableQueryImpl(name, context, true); } - /// Get the CREATE DICTIONARY query for the dictionary. Returns nullptr if dictionary doesn't exists. - ASTPtr tryGetCreateDictionaryQuery(const String & name) const noexcept - { - return getCreateDictionaryQueryImpl(name, false); - } - - ASTPtr getCreateDictionaryQuery(const String & name) const - { - return getCreateDictionaryQueryImpl(name, true); - } - - virtual Poco::AutoPtr getDictionaryConfiguration(const String & /*name*/) const - { - throw Exception(getEngineName() + ": getDictionaryConfiguration() is not supported", ErrorCodes::NOT_IMPLEMENTED); - } - /// Get the CREATE DATABASE query for current database. virtual ASTPtr getCreateDatabaseQuery() const = 0; @@ -364,13 +274,6 @@ protected: return nullptr; } - virtual ASTPtr getCreateDictionaryQueryImpl(const String & /*name*/, bool throw_on_error) const - { - if (throw_on_error) - throw Exception("There is no SHOW CREATE DICTIONARY query for Database" + getEngineName(), ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY); - return nullptr; - } - mutable std::mutex mutex; String database_name; }; diff --git a/src/Databases/ya.make b/src/Databases/ya.make index 8bd3f291a64..e1b61ee4d26 100644 --- a/src/Databases/ya.make +++ b/src/Databases/ya.make @@ -19,7 +19,6 @@ SRCS( DatabaseReplicated.cpp DatabaseReplicatedSettings.cpp DatabaseReplicatedWorker.cpp - DatabaseWithDictionaries.cpp DatabasesCommon.cpp MySQL/ConnectionMySQLSettings.cpp MySQL/DatabaseConnectionMySQL.cpp diff --git a/src/Disks/CMakeLists.txt b/src/Disks/CMakeLists.txt index 65172356645..83bbe418246 100644 --- a/src/Disks/CMakeLists.txt +++ b/src/Disks/CMakeLists.txt @@ -1,3 +1,3 @@ -if (ENABLE_TESTS) - add_subdirectory (tests) -endif () +if (ENABLE_EXAMPLES) + add_subdirectory(examples) +endif() diff --git a/src/Dictionaries/tests/CMakeLists.txt b/src/Disks/examples/CMakeLists.txt similarity index 100% rename from src/Dictionaries/tests/CMakeLists.txt rename to src/Disks/examples/CMakeLists.txt diff --git a/src/Formats/CMakeLists.txt b/src/Formats/CMakeLists.txt index 4929c141fe3..0a342917073 100644 --- a/src/Formats/CMakeLists.txt +++ b/src/Formats/CMakeLists.txt @@ -1,5 +1,5 @@ configure_file(config_formats.h.in ${ConfigIncludePath}/config_formats.h) -if (ENABLE_TESTS) - add_subdirectory (tests) -endif () +if (ENABLE_EXAMPLES) + add_subdirectory(examples) +endif() diff --git a/src/Formats/tests/CMakeLists.txt b/src/Formats/examples/CMakeLists.txt similarity index 100% rename from src/Formats/tests/CMakeLists.txt rename to src/Formats/examples/CMakeLists.txt diff --git a/src/Formats/tests/tab_separated_streams.cpp b/src/Formats/examples/tab_separated_streams.cpp similarity index 100% rename from src/Formats/tests/tab_separated_streams.cpp rename to src/Formats/examples/tab_separated_streams.cpp diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 27a6fa76de8..7e9e953eabe 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -66,8 +66,8 @@ if (USE_FASTOPS) target_include_directories (clickhouse_functions SYSTEM PRIVATE ${FASTOPS_INCLUDE_DIR}) endif () -if (ENABLE_TESTS) - add_subdirectory (tests) +if (ENABLE_EXAMPLES) + add_subdirectory(examples) endif () if (USE_EMBEDDED_COMPILER) diff --git a/src/Disks/tests/CMakeLists.txt b/src/Functions/examples/CMakeLists.txt similarity index 100% rename from src/Disks/tests/CMakeLists.txt rename to src/Functions/examples/CMakeLists.txt diff --git a/src/Functions/timezoneOffset.cpp b/src/Functions/timezoneOffset.cpp index 5acdb105b2b..296603ce0a4 100644 --- a/src/Functions/timezoneOffset.cpp +++ b/src/Functions/timezoneOffset.cpp @@ -12,6 +12,7 @@ using FunctiontimezoneOffset = FunctionDateOrDateTimeToSomething(); + factory.registerAlias("timeZoneOffset", "timezoneOffset"); } } diff --git a/src/IO/CMakeLists.txt b/src/IO/CMakeLists.txt index 65172356645..f676f415eea 100644 --- a/src/IO/CMakeLists.txt +++ b/src/IO/CMakeLists.txt @@ -1,3 +1,3 @@ -if (ENABLE_TESTS) - add_subdirectory (tests) +if (ENABLE_EXAMPLES) + add_subdirectory (examples) endif () diff --git a/src/IO/TimeoutSetter.cpp b/src/IO/TimeoutSetter.cpp index f06cafecff8..5a87883824f 100644 --- a/src/IO/TimeoutSetter.cpp +++ b/src/IO/TimeoutSetter.cpp @@ -2,8 +2,10 @@ #include + namespace DB { + TimeoutSetter::TimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & send_timeout_, const Poco::Timespan & receive_timeout_, @@ -32,10 +34,12 @@ TimeoutSetter::~TimeoutSetter() socket.setSendTimeout(old_send_timeout); socket.setReceiveTimeout(old_receive_timeout); } - catch (std::exception & e) + catch (...) { - // Sometimes caught on macos - LOG_ERROR(&Poco::Logger::get("Client"), "TimeoutSetter: Can't reset timeouts: {}", e.what()); + /// Sometimes caught on Mac OS X. This message can be safely ignored. + /// If you are developer using Mac, please debug this error message by yourself. + tryLogCurrentException("Client", "TimeoutSetter: Can't reset timeouts"); } } + } diff --git a/src/IO/WriteBufferFromPocoSocket.cpp b/src/IO/WriteBufferFromPocoSocket.cpp index 7338bcabdb5..78705857ec4 100644 --- a/src/IO/WriteBufferFromPocoSocket.cpp +++ b/src/IO/WriteBufferFromPocoSocket.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index b9497b6f87e..0f8b5adee1a 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -446,6 +447,46 @@ inline void writeJSONString(const String & s, WriteBuffer & buf, const FormatSet writeJSONString(StringRef{s}, buf, settings); } +template +void writeJSONNumber(T x, WriteBuffer & ostr, const FormatSettings & settings) +{ + bool is_finite = isFinite(x); + + const bool need_quote = (is_integer_v && (sizeof(T) >= 8) && settings.json.quote_64bit_integers) + || (settings.json.quote_denormals && !is_finite); + + if (need_quote) + writeChar('"', ostr); + + if (is_finite) + writeText(x, ostr); + else if (!settings.json.quote_denormals) + writeCString("null", ostr); + else + { + if constexpr (std::is_floating_point_v) + { + if (std::signbit(x)) + { + if (isNaN(x)) + writeCString("-nan", ostr); + else + writeCString("-inf", ostr); + } + else + { + if (isNaN(x)) + writeCString("nan", ostr); + else + writeCString("inf", ostr); + } + } + } + + if (need_quote) + writeChar('"', ostr); +} + template void writeAnyEscapedString(const String & s, WriteBuffer & buf) diff --git a/src/IO/tests/CMakeLists.txt b/src/IO/examples/CMakeLists.txt similarity index 100% rename from src/IO/tests/CMakeLists.txt rename to src/IO/examples/CMakeLists.txt diff --git a/src/IO/tests/dragonbox_test.cpp b/src/IO/examples/dragonbox_test.cpp similarity index 100% rename from src/IO/tests/dragonbox_test.cpp rename to src/IO/examples/dragonbox_test.cpp diff --git a/src/IO/tests/hashing_buffer.h b/src/IO/examples/hashing_buffer.h similarity index 100% rename from src/IO/tests/hashing_buffer.h rename to src/IO/examples/hashing_buffer.h diff --git a/src/IO/tests/hashing_read_buffer.cpp b/src/IO/examples/hashing_read_buffer.cpp similarity index 100% rename from src/IO/tests/hashing_read_buffer.cpp rename to src/IO/examples/hashing_read_buffer.cpp diff --git a/src/IO/tests/hashing_write_buffer.cpp b/src/IO/examples/hashing_write_buffer.cpp similarity index 100% rename from src/IO/tests/hashing_write_buffer.cpp rename to src/IO/examples/hashing_write_buffer.cpp diff --git a/src/IO/tests/io_operators.cpp b/src/IO/examples/io_operators.cpp similarity index 100% rename from src/IO/tests/io_operators.cpp rename to src/IO/examples/io_operators.cpp diff --git a/src/IO/tests/limit_read_buffer.cpp b/src/IO/examples/limit_read_buffer.cpp similarity index 100% rename from src/IO/tests/limit_read_buffer.cpp rename to src/IO/examples/limit_read_buffer.cpp diff --git a/src/IO/tests/limit_read_buffer2.cpp b/src/IO/examples/limit_read_buffer2.cpp similarity index 100% rename from src/IO/tests/limit_read_buffer2.cpp rename to src/IO/examples/limit_read_buffer2.cpp diff --git a/src/IO/tests/lzma_buffers.cpp b/src/IO/examples/lzma_buffers.cpp similarity index 100% rename from src/IO/tests/lzma_buffers.cpp rename to src/IO/examples/lzma_buffers.cpp diff --git a/src/IO/tests/o_direct_and_dirty_pages.cpp b/src/IO/examples/o_direct_and_dirty_pages.cpp similarity index 100% rename from src/IO/tests/o_direct_and_dirty_pages.cpp rename to src/IO/examples/o_direct_and_dirty_pages.cpp diff --git a/src/IO/tests/parse_date_time_best_effort.cpp b/src/IO/examples/parse_date_time_best_effort.cpp similarity index 100% rename from src/IO/tests/parse_date_time_best_effort.cpp rename to src/IO/examples/parse_date_time_best_effort.cpp diff --git a/src/IO/tests/parse_int_perf.cpp b/src/IO/examples/parse_int_perf.cpp similarity index 100% rename from src/IO/tests/parse_int_perf.cpp rename to src/IO/examples/parse_int_perf.cpp diff --git a/src/IO/tests/parse_int_perf2.cpp b/src/IO/examples/parse_int_perf2.cpp similarity index 100% rename from src/IO/tests/parse_int_perf2.cpp rename to src/IO/examples/parse_int_perf2.cpp diff --git a/src/IO/tests/read_buffer.cpp b/src/IO/examples/read_buffer.cpp similarity index 100% rename from src/IO/tests/read_buffer.cpp rename to src/IO/examples/read_buffer.cpp diff --git a/src/IO/tests/read_buffer_aio.cpp b/src/IO/examples/read_buffer_aio.cpp similarity index 100% rename from src/IO/tests/read_buffer_aio.cpp rename to src/IO/examples/read_buffer_aio.cpp diff --git a/src/IO/tests/read_buffer_perf.cpp b/src/IO/examples/read_buffer_perf.cpp similarity index 100% rename from src/IO/tests/read_buffer_perf.cpp rename to src/IO/examples/read_buffer_perf.cpp diff --git a/src/IO/tests/read_escaped_string.cpp b/src/IO/examples/read_escaped_string.cpp similarity index 100% rename from src/IO/tests/read_escaped_string.cpp rename to src/IO/examples/read_escaped_string.cpp diff --git a/src/IO/tests/read_float_perf.cpp b/src/IO/examples/read_float_perf.cpp similarity index 100% rename from src/IO/tests/read_float_perf.cpp rename to src/IO/examples/read_float_perf.cpp diff --git a/src/IO/tests/read_write_int.cpp b/src/IO/examples/read_write_int.cpp similarity index 100% rename from src/IO/tests/read_write_int.cpp rename to src/IO/examples/read_write_int.cpp diff --git a/src/IO/tests/valid_utf8.cpp b/src/IO/examples/valid_utf8.cpp similarity index 100% rename from src/IO/tests/valid_utf8.cpp rename to src/IO/examples/valid_utf8.cpp diff --git a/src/IO/tests/valid_utf8_perf.cpp b/src/IO/examples/valid_utf8_perf.cpp similarity index 100% rename from src/IO/tests/valid_utf8_perf.cpp rename to src/IO/examples/valid_utf8_perf.cpp diff --git a/src/IO/tests/var_uint.cpp b/src/IO/examples/var_uint.cpp similarity index 100% rename from src/IO/tests/var_uint.cpp rename to src/IO/examples/var_uint.cpp diff --git a/src/IO/tests/write_buffer.cpp b/src/IO/examples/write_buffer.cpp similarity index 100% rename from src/IO/tests/write_buffer.cpp rename to src/IO/examples/write_buffer.cpp diff --git a/src/IO/tests/write_buffer_perf.cpp b/src/IO/examples/write_buffer_perf.cpp similarity index 100% rename from src/IO/tests/write_buffer_perf.cpp rename to src/IO/examples/write_buffer_perf.cpp diff --git a/src/IO/tests/write_int.cpp b/src/IO/examples/write_int.cpp similarity index 100% rename from src/IO/tests/write_int.cpp rename to src/IO/examples/write_int.cpp diff --git a/src/IO/tests/zlib_buffers.cpp b/src/IO/examples/zlib_buffers.cpp similarity index 100% rename from src/IO/tests/zlib_buffers.cpp rename to src/IO/examples/zlib_buffers.cpp diff --git a/src/IO/tests/zlib_ng_bug.cpp b/src/IO/examples/zlib_ng_bug.cpp similarity index 100% rename from src/IO/tests/zlib_ng_bug.cpp rename to src/IO/examples/zlib_ng_bug.cpp diff --git a/src/IO/tests/zstd_buffers.cpp b/src/IO/examples/zstd_buffers.cpp similarity index 100% rename from src/IO/tests/zstd_buffers.cpp rename to src/IO/examples/zstd_buffers.cpp diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 190332712ff..d9a8bfaf462 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -13,6 +13,7 @@ #include #include +#include namespace DB { @@ -27,6 +28,47 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } +const char * ActionsDAG::typeToString(ActionsDAG::ActionType type) +{ + switch (type) + { + case ActionType::INPUT: + return "Input"; + case ActionType::COLUMN: + return "Column"; + case ActionType::ALIAS: + return "Alias"; + case ActionType::ARRAY_JOIN: + return "ArrayJoin"; + case ActionType::FUNCTION: + return "Function"; + } + + __builtin_unreachable(); +} + +void ActionsDAG::Node::toTree(JSONBuilder::JSONMap & map) const +{ + map.add("Node Type", ActionsDAG::typeToString(type)); + + if (result_type) + map.add("Result Type", result_type->getName()); + + if (!result_name.empty()) + map.add("Result Type", ActionsDAG::typeToString(type)); + + if (column) + map.add("Column", column->getName()); + + if (function_base) + map.add("Function", function_base->getName()); + else if (function_builder) + map.add("Function", function_builder->getName()); + + if (type == ActionType::FUNCTION) + map.add("Compiled", is_function_compiled); +} + ActionsDAG::ActionsDAG(const NamesAndTypesList & inputs_) { diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index bdfb4644f5b..645f1aaeeb7 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -29,6 +29,14 @@ using DataTypePtr = std::shared_ptr; class CompiledExpressionCache; +namespace JSONBuilder +{ + class JSONMap; + + class IItem; + using ItemPtr = std::unique_ptr; +} + /// Directed acyclic graph of expressions. /// This is an intermediate representation of actions which is usually built from expression list AST. /// Node of DAG describe calculation of a single column with known type, name, and constant value (if applicable). @@ -55,6 +63,8 @@ public: FUNCTION, }; + static const char * typeToString(ActionType type); + struct Node; using NodeRawPtrs = std::vector; using NodeRawConstPtrs = std::vector; @@ -81,6 +91,8 @@ public: /// Some functions like `ignore()` always return constant but can't be replaced by constant it. /// We calculate such constants in order to avoid unnecessary materialization, but prohibit it's folding. bool allow_constant_folding = true; + + void toTree(JSONBuilder::JSONMap & map) const; }; /// NOTE: std::list is an implementation detail. diff --git a/src/Interpreters/AggregateDescription.cpp b/src/Interpreters/AggregateDescription.cpp index e483eb1b7a1..ca10878a4ae 100644 --- a/src/Interpreters/AggregateDescription.cpp +++ b/src/Interpreters/AggregateDescription.cpp @@ -2,6 +2,8 @@ #include #include +#include + namespace DB { @@ -99,4 +101,50 @@ void AggregateDescription::explain(WriteBuffer & out, size_t indent) const } } +void AggregateDescription::explain(JSONBuilder::JSONMap & map) const +{ + map.add("Name", column_name); + + if (function) + { + auto function_map = std::make_unique(); + + function_map->add("Name", function->getName()); + + const auto & params = function->getParameters(); + if (!params.empty()) + { + auto params_array = std::make_unique(); + for (const auto & param : params) + params_array->add(applyVisitor(FieldVisitorToString(), param)); + + function_map->add("Parameters", std::move(params_array)); + } + + auto args_array = std::make_unique(); + for (const auto & type : function->getArgumentTypes()) + args_array->add(type->getName()); + + function_map->add("Argument Types", std::move(args_array)); + function_map->add("Result Type", function->getReturnType()->getName()); + + map.add("Function", std::move(function_map)); + } + + auto args_array = std::make_unique(); + for (const auto & name : argument_names) + args_array->add(name); + + map.add("Arguments", std::move(args_array)); + + if (!arguments.empty()) + { + auto args_pos_array = std::make_unique(); + for (auto pos : arguments) + args_pos_array->add(pos); + + map.add("Argument Positions", std::move(args_pos_array)); + } +} + } diff --git a/src/Interpreters/AggregateDescription.h b/src/Interpreters/AggregateDescription.h index 3af0dc38586..12c14f7a57c 100644 --- a/src/Interpreters/AggregateDescription.h +++ b/src/Interpreters/AggregateDescription.h @@ -5,10 +5,11 @@ #include #include - namespace DB { +namespace JSONBuilder { class JSONMap; } + struct AggregateDescription { AggregateFunctionPtr function; @@ -18,6 +19,7 @@ struct AggregateDescription String column_name; /// What name to use for a column with aggregate function values void explain(WriteBuffer & out, size_t indent) const; /// Get description for EXPLAIN query. + void explain(JSONBuilder::JSONMap & map) const; }; using AggregateDescriptions = std::vector; diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 0e6a19d7b04..984b55e6e60 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -178,6 +179,38 @@ void Aggregator::Params::explain(WriteBuffer & out, size_t indent) const } } +void Aggregator::Params::explain(JSONBuilder::JSONMap & map) const +{ + const auto & header = src_header ? src_header + : intermediate_header; + + auto keys_array = std::make_unique(); + + for (auto key : keys) + { + if (key >= header.columns()) + keys_array->add(""); + else + keys_array->add(header.getByPosition(key).name); + } + + map.add("Keys", std::move(keys_array)); + + if (!aggregates.empty()) + { + auto aggregates_array = std::make_unique(); + + for (const auto & aggregate : aggregates) + { + auto aggregate_map = std::make_unique(); + aggregate.explain(*aggregate_map); + aggregates_array->add(std::move(aggregate_map)); + } + + map.add("Aggregates", std::move(aggregates_array)); + } +} + Aggregator::Aggregator(const Params & params_) : params(params_) { @@ -1769,6 +1802,8 @@ void NO_INLINE Aggregator::mergeStreamsImplCase( /// For all rows. size_t rows = block.rows(); + std::unique_ptr places(new AggregateDataPtr[rows]); + for (size_t i = 0; i < rows; ++i) { AggregateDataPtr aggregate_data = nullptr; @@ -1797,18 +1832,17 @@ void NO_INLINE Aggregator::mergeStreamsImplCase( /// aggregate_date == nullptr means that the new key did not fit in the hash table because of no_more_keys. - /// If the key does not fit, and the data does not need to be aggregated into a separate row, then there's nothing to do. - if (!aggregate_data && !overflow_row) - continue; - AggregateDataPtr value = aggregate_data ? aggregate_data : overflow_row; + places[i] = value; + } + for (size_t j = 0; j < params.aggregates_size; ++j) + { /// Merge state of aggregate functions. - for (size_t j = 0; j < params.aggregates_size; ++j) - aggregate_functions[j]->merge( - value + offsets_of_aggregate_states[j], - (*aggregate_columns[j])[i], - aggregates_pool); + aggregate_functions[j]->mergeBatch( + rows, places.get(), offsets_of_aggregate_states[j], + aggregate_columns[j]->data(), + aggregates_pool); } /// Early release memory. diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index b335eb88b06..1b5ed75b0d0 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -950,6 +950,7 @@ public: /// Returns keys and aggregated for EXPLAIN query void explain(WriteBuffer & out, size_t indent) const; + void explain(JSONBuilder::JSONMap & map) const; }; explicit Aggregator(const Params & params_); diff --git a/src/Interpreters/CMakeLists.txt b/src/Interpreters/CMakeLists.txt index 65172356645..1068fee4cea 100644 --- a/src/Interpreters/CMakeLists.txt +++ b/src/Interpreters/CMakeLists.txt @@ -1,3 +1,7 @@ if (ENABLE_TESTS) - add_subdirectory (tests) -endif () + add_subdirectory(tests) +endif() + +if (ENABLE_EXAMPLES) + add_subdirectory(examples) +endif() \ No newline at end of file diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index d88b87a73d4..9dd3830e136 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -630,8 +630,10 @@ std::unique_lock DatabaseCatalog::getExclusiveDDLGuardForData bool DatabaseCatalog::isDictionaryExist(const StorageID & table_id) const { - auto db = tryGetDatabase(table_id.getDatabaseName()); - return db && db->isDictionaryExist(table_id.getTableName()); + auto storage = tryGetTable(table_id, getContext()); + bool storage_is_dictionary = storage && storage->isDictionary(); + + return storage_is_dictionary; } StoragePtr DatabaseCatalog::getTable(const StorageID & table_id, ContextPtr local_context) const diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 86fb7bac671..d1f5b7cd896 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #if defined(MEMORY_SANITIZER) #include @@ -258,6 +259,29 @@ std::string ExpressionActions::Action::toString() const return out.str(); } +JSONBuilder::ItemPtr ExpressionActions::Action::toTree() const +{ + auto map = std::make_unique(); + + if (node) + node->toTree(*map); + + auto args = std::make_unique(); + auto dropped_args = std::make_unique(); + for (auto arg : arguments) + { + args->add(arg.pos); + if (!arg.needed_later) + dropped_args->add(arg.pos); + } + + map->add("Arguments", std::move(args)); + map->add("Removed Arguments", std::move(dropped_args)); + map->add("Result", result_position); + + return map; +} + void ExpressionActions::checkLimits(const ColumnsWithTypeAndName & columns) const { if (settings.max_temporary_non_const_columns) @@ -567,6 +591,50 @@ std::string ExpressionActions::dumpActions() const return ss.str(); } +JSONBuilder::ItemPtr ExpressionActions::toTree() const +{ + auto inputs_array = std::make_unique(); + + for (const auto & input_column : required_columns) + { + auto map = std::make_unique(); + map->add("Name", input_column.name); + if (input_column.type) + map->add("Type", input_column.type->getName()); + + inputs_array->add(std::move(map)); + } + + auto outputs_array = std::make_unique(); + + for (const auto & output_column : sample_block) + { + auto map = std::make_unique(); + map->add("Name", output_column.name); + if (output_column.type) + map->add("Type", output_column.type->getName()); + + outputs_array->add(std::move(map)); + } + + auto actions_array = std::make_unique(); + for (const auto & action : actions) + actions_array->add(action.toTree()); + + auto positions_array = std::make_unique(); + for (auto pos : result_positions) + positions_array->add(pos); + + auto map = std::make_unique(); + map->add("Inputs", std::move(inputs_array)); + map->add("Actions", std::move(actions_array)); + map->add("Outputs", std::move(outputs_array)); + map->add("Positions", std::move(positions_array)); + map->add("Project Input", actions_dag->isInputProjected()); + + return map; +} + bool ExpressionActions::checkColumnIsAlwaysFalse(const String & column_name) const { /// Check has column in (empty set). diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h index 40a4cdcbc02..eb3b60c1ffb 100644 --- a/src/Interpreters/ExpressionActions.h +++ b/src/Interpreters/ExpressionActions.h @@ -58,6 +58,7 @@ public: size_t result_position; std::string toString() const; + JSONBuilder::ItemPtr toTree() const; }; using Actions = std::vector; @@ -108,6 +109,7 @@ public: const Block & getSampleBlock() const { return sample_block; } std::string dumpActions() const; + JSONBuilder::ItemPtr toTree() const; static std::string getSmallestColumn(const NamesAndTypesList & columns); diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 1fa0f4eab7c..ee576c8deda 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -755,11 +755,11 @@ static ExpressionActionsPtr createJoinedBlockActions(ContextPtr context, const T static bool allowDictJoin(StoragePtr joined_storage, ContextPtr context, String & dict_name, String & key_name) { - const auto * dict = dynamic_cast(joined_storage.get()); - if (!dict) + if (!joined_storage->isDictionary()) return false; - dict_name = dict->dictionaryName(); + StorageDictionary & storage_dictionary = static_cast(*joined_storage); + dict_name = storage_dictionary.getDictionaryName(); auto dictionary = context->getExternalDictionariesLoader().getDictionary(dict_name, context); if (!dictionary) return false; diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 1fc0f95c846..7fa00cdf747 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -32,7 +32,6 @@ ExternalDictionariesLoader::ExternalDictionariesLoader(ContextPtr global_context enablePeriodicUpdates(true); } - ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config, const std::string & repository_name) const diff --git a/src/Interpreters/ExternalLoaderDatabaseConfigRepository.cpp b/src/Interpreters/ExternalLoaderDatabaseConfigRepository.cpp deleted file mode 100644 index a8b65a18983..00000000000 --- a/src/Interpreters/ExternalLoaderDatabaseConfigRepository.cpp +++ /dev/null @@ -1,89 +0,0 @@ -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int UNKNOWN_DICTIONARY; -} - - -namespace -{ - String trimDatabaseName(const std::string & loadable_definition_name, const String & database_name, - const IDatabase & database, ContextPtr global_context) - { - bool is_atomic_database = database.getUUID() != UUIDHelpers::Nil; - if (is_atomic_database) - { - /// We do not know actual database and dictionary names here - auto dict_id = StorageID::createEmpty(); - dict_id.uuid = parseFromString(loadable_definition_name); - assert(dict_id.uuid != UUIDHelpers::Nil); - /// Get associated StorageDictionary by UUID - auto table = DatabaseCatalog::instance().getTable(dict_id, global_context); - auto dict_id_with_names = table->getStorageID(); - return dict_id_with_names.table_name; - } - - if (!startsWith(loadable_definition_name, database_name)) - throw Exception(ErrorCodes::UNKNOWN_DICTIONARY, - "Loadable '{}' is not from database '{}'", loadable_definition_name, database_name); - /// dbname.loadable_name - ///--> remove <--- - return loadable_definition_name.substr(database_name.length() + 1); - } -} - - -ExternalLoaderDatabaseConfigRepository::ExternalLoaderDatabaseConfigRepository(IDatabase & database_, ContextPtr context_) - : WithContext(context_->getGlobalContext()) - , database_name(database_.getDatabaseName()) - , database(database_) -{ -} - -LoadablesConfigurationPtr ExternalLoaderDatabaseConfigRepository::load(const std::string & loadable_definition_name) -{ - auto dict_name = trimDatabaseName(loadable_definition_name, database_name, database, getContext()); - return database.getDictionaryConfiguration(dict_name); -} - -bool ExternalLoaderDatabaseConfigRepository::exists(const std::string & loadable_definition_name) -{ - auto dict_name = trimDatabaseName(loadable_definition_name, database_name, database, getContext()); - return database.isDictionaryExist(dict_name); -} - -Poco::Timestamp ExternalLoaderDatabaseConfigRepository::getUpdateTime(const std::string & loadable_definition_name) -{ - auto dict_name = trimDatabaseName(loadable_definition_name, database_name, database, getContext()); - return database.getObjectMetadataModificationTime(dict_name); -} - -std::set ExternalLoaderDatabaseConfigRepository::getAllLoadablesDefinitionNames() -{ - std::set result; - auto itr = database.getDictionariesIterator(); - bool is_atomic_database = database.getUUID() != UUIDHelpers::Nil; - while (itr && itr->isValid()) - { - if (is_atomic_database) - { - assert(itr->uuid() != UUIDHelpers::Nil); - result.insert(toString(itr->uuid())); - } - else - result.insert(database_name + "." + itr->name()); - itr->next(); - } - return result; -} - -} diff --git a/src/Interpreters/ExternalLoaderDatabaseConfigRepository.h b/src/Interpreters/ExternalLoaderDatabaseConfigRepository.h index baa99bc5610..b8dd6e278ad 100644 --- a/src/Interpreters/ExternalLoaderDatabaseConfigRepository.h +++ b/src/Interpreters/ExternalLoaderDatabaseConfigRepository.h @@ -14,7 +14,7 @@ class ExternalLoaderDatabaseConfigRepository : public IExternalLoaderConfigRepos public: ExternalLoaderDatabaseConfigRepository(IDatabase & database_, ContextPtr global_context_); - const std::string & getName() const override { return database_name; } + std::string getName() const override { return database_name; } std::set getAllLoadablesDefinitionNames() override; diff --git a/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.cpp b/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.cpp new file mode 100644 index 00000000000..86f5a9ded0a --- /dev/null +++ b/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.cpp @@ -0,0 +1,39 @@ +#include "ExternalLoaderDictionaryStorageConfigRepository.h" + +#include +#include + +namespace DB +{ + +ExternalLoaderDictionaryStorageConfigRepository::ExternalLoaderDictionaryStorageConfigRepository(const StorageDictionary & dictionary_storage_) + : dictionary_storage(dictionary_storage_) +{ +} + +std::string ExternalLoaderDictionaryStorageConfigRepository::getName() const +{ + return dictionary_storage.getStorageID().getInternalDictionaryName(); +} + +std::set ExternalLoaderDictionaryStorageConfigRepository::getAllLoadablesDefinitionNames() +{ + return { getName() }; +} + +bool ExternalLoaderDictionaryStorageConfigRepository::exists(const std::string & loadable_definition_name) +{ + return getName() == loadable_definition_name; +} + +Poco::Timestamp ExternalLoaderDictionaryStorageConfigRepository::getUpdateTime(const std::string &) +{ + return dictionary_storage.getUpdateTime(); +} + +LoadablesConfigurationPtr ExternalLoaderDictionaryStorageConfigRepository::load(const std::string &) +{ + return dictionary_storage.getConfiguration(); +} + +} diff --git a/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.h b/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.h new file mode 100644 index 00000000000..06d2b0faf75 --- /dev/null +++ b/src/Interpreters/ExternalLoaderDictionaryStorageConfigRepository.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +namespace DB +{ + +class StorageDictionary; + +class ExternalLoaderDictionaryStorageConfigRepository : public IExternalLoaderConfigRepository +{ +public: + explicit ExternalLoaderDictionaryStorageConfigRepository(const StorageDictionary & dictionary_storage_); + + std::string getName() const override; + + std::set getAllLoadablesDefinitionNames() override; + + bool exists(const std::string & loadable_definition_name) override; + + Poco::Timestamp getUpdateTime(const std::string & loadable_definition_name) override; + + LoadablesConfigurationPtr load(const std::string & loadable_definition_name) override; + +private: + const StorageDictionary & dictionary_storage; +}; + +} diff --git a/src/Interpreters/ExternalLoaderTempConfigRepository.h b/src/Interpreters/ExternalLoaderTempConfigRepository.h index 46e2eb846e9..d042f310ed1 100644 --- a/src/Interpreters/ExternalLoaderTempConfigRepository.h +++ b/src/Interpreters/ExternalLoaderTempConfigRepository.h @@ -13,7 +13,7 @@ class ExternalLoaderTempConfigRepository : public IExternalLoaderConfigRepositor public: ExternalLoaderTempConfigRepository(const String & repository_name_, const String & path_, const LoadablesConfigurationPtr & config_); - const String & getName() const override { return name; } + String getName() const override { return name; } bool isTemporary() const override { return true; } std::set getAllLoadablesDefinitionNames() override; diff --git a/src/Interpreters/ExternalLoaderXMLConfigRepository.h b/src/Interpreters/ExternalLoaderXMLConfigRepository.h index dd689856300..76f51f04397 100644 --- a/src/Interpreters/ExternalLoaderXMLConfigRepository.h +++ b/src/Interpreters/ExternalLoaderXMLConfigRepository.h @@ -15,7 +15,7 @@ class ExternalLoaderXMLConfigRepository : public IExternalLoaderConfigRepository public: ExternalLoaderXMLConfigRepository(const Poco::Util::AbstractConfiguration & main_config_, const std::string & config_key_); - const String & getName() const override { return name; } + std::string getName() const override { return name; } /// Return set of .xml files from path in main_config (config_key) std::set getAllLoadablesDefinitionNames() override; diff --git a/src/Interpreters/IExternalLoaderConfigRepository.h b/src/Interpreters/IExternalLoaderConfigRepository.h index 866aa0b877f..7d2e4799810 100644 --- a/src/Interpreters/IExternalLoaderConfigRepository.h +++ b/src/Interpreters/IExternalLoaderConfigRepository.h @@ -23,7 +23,7 @@ class IExternalLoaderConfigRepository { public: /// Returns the name of the repository. - virtual const std::string & getName() const = 0; + virtual std::string getName() const = 0; /// Whether this repository is temporary: /// it's created and destroyed while executing the same query. diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index d155a0eea4b..d45d02243fb 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -71,6 +71,7 @@ namespace DB namespace ErrorCodes { extern const int TABLE_ALREADY_EXISTS; + extern const int DICTIONARY_ALREADY_EXISTS; extern const int EMPTY_LIST_OF_COLUMNS_PASSED; extern const int INCORRECT_QUERY; extern const int UNKNOWN_DATABASE_ENGINE; @@ -79,7 +80,6 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int BAD_DATABASE_FOR_TEMPORARY_TABLE; extern const int SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY; - extern const int DICTIONARY_ALREADY_EXISTS; extern const int ILLEGAL_SYNTAX_FOR_DATA_TYPE; extern const int ILLEGAL_COLUMN; extern const int LOGICAL_ERROR; @@ -551,6 +551,10 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::setProperties(AS properties.columns = table_function->getActualTableStructure(getContext()); assert(!properties.columns.empty()); } + else if (create.is_dictionary) + { + return {}; + } else throw Exception("Incorrect CREATE query: required list of column descriptions or AS section or SELECT.", ErrorCodes::INCORRECT_QUERY); @@ -837,11 +841,20 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) // Table SQL definition is available even if the table is detached (even permanently) auto query = database->getCreateTableQuery(create.table, getContext()); - create = query->as(); // Copy the saved create query, but use ATTACH instead of CREATE - if (create.is_dictionary) + auto create_query = query->as(); + + if (!create.is_dictionary && create_query.is_dictionary) throw Exception(ErrorCodes::INCORRECT_QUERY, - "Cannot ATTACH TABLE {}.{}, it is a Dictionary", - backQuoteIfNeed(database_name), backQuoteIfNeed(create.table)); + "Cannot ATTACH TABLE {}.{}, it is a Dictionary", + backQuoteIfNeed(database_name), backQuoteIfNeed(create.table)); + + if (create.is_dictionary && !create_query.is_dictionary) + throw Exception(ErrorCodes::INCORRECT_QUERY, + "Cannot ATTACH DICTIONARY {}.{}, it is a Table", + backQuoteIfNeed(database_name), backQuoteIfNeed(create.table)); + + create = create_query; // Copy the saved create query, but use ATTACH instead of CREATE + create.attach = true; create.attach_short_syntax = true; create.if_not_exists = if_not_exists; @@ -947,6 +960,9 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, database = DatabaseCatalog::instance().getDatabase(create.database); assertOrSetUUID(create, database); + String storage_name = create.is_dictionary ? "Dictionary" : "Table"; + auto storage_already_exists_error_code = create.is_dictionary ? ErrorCodes::DICTIONARY_ALREADY_EXISTS : ErrorCodes::TABLE_ALREADY_EXISTS; + /// Table can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard. if (database->isTableExist(create.table, getContext())) { @@ -966,12 +982,13 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, interpreter.execute(); } else - throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists.", backQuoteIfNeed(create.database), backQuoteIfNeed(create.table)); + throw Exception(storage_already_exists_error_code, + "{} {}.{} already exists.", storage_name, backQuoteIfNeed(create.database), backQuoteIfNeed(create.table)); } data_path = database->getTableDataPath(create); if (!create.attach && !data_path.empty() && fs::exists(fs::path{getContext()->getPath()} / data_path)) - throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Directory for table data {} already exists", String(data_path)); + throw Exception(storage_already_exists_error_code, "Directory for {} data {} already exists", Poco::toLower(storage_name), String(data_path)); } else { @@ -1123,56 +1140,6 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create) return {}; } -BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create) -{ - String dictionary_name = create.table; - - create.database = getContext()->resolveDatabase(create.database); - const String & database_name = create.database; - - auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, dictionary_name); - DatabasePtr database = DatabaseCatalog::instance().getDatabase(database_name); - - if (typeid_cast(database.get()) - && getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY) - { - if (!create.attach) - assertOrSetUUID(create, database); - guard->releaseTableLock(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext()); - } - - if (database->isDictionaryExist(dictionary_name)) - { - /// TODO Check structure of dictionary - if (create.if_not_exists) - return {}; - else - throw Exception( - "Dictionary " + database_name + "." + dictionary_name + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS); - } - - if (create.attach) - { - auto query = DatabaseCatalog::instance().getDatabase(database_name)->getCreateDictionaryQuery(dictionary_name); - create = query->as(); - create.attach = true; - } - - assertOrSetUUID(create, database); - - if (create.attach) - { - auto config = getDictionaryConfigurationFromAST(create, getContext()); - auto modification_time = database->getObjectMetadataModificationTime(dictionary_name); - database->attachDictionary(dictionary_name, DictionaryAttachInfo{query_ptr, config, modification_time}); - } - else - database->createDictionary(getContext(), dictionary_name, query_ptr); - - return {}; -} - void InterpreterCreateQuery::prepareOnClusterQuery(ASTCreateQuery & create, ContextPtr local_context, const String & cluster_name) { if (create.attach) @@ -1237,10 +1204,8 @@ BlockIO InterpreterCreateQuery::execute() /// CREATE|ATTACH DATABASE if (!create.database.empty() && create.table.empty()) return createDatabase(create); - else if (!create.is_dictionary) - return createTable(create); else - return createDictionary(create); + return createTable(create); } diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index f674bf19123..f665ec85d46 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -70,7 +70,6 @@ private: BlockIO createDatabase(ASTCreateQuery & create); BlockIO createTable(ASTCreateQuery & create); - BlockIO createDictionary(ASTCreateQuery & create); /// Calculate list of columns, constraints, indices, etc... of table. Rewrite query in canonical way. TableProperties setProperties(ASTCreateQuery & create) const; diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 8ac6a0ae3a8..0b4efe4f978 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -31,7 +31,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int SYNTAX_ERROR; extern const int UNKNOWN_TABLE; - extern const int UNKNOWN_DICTIONARY; extern const int NOT_IMPLEMENTED; extern const int INCORRECT_QUERY; } @@ -58,14 +57,7 @@ BlockIO InterpreterDropQuery::execute() drop.no_delay = true; if (!drop.table.empty()) - { - if (!drop.is_dictionary) - return executeToTable(drop); - else if (drop.permanently && drop.kind == ASTDropQuery::Kind::Detach) - throw Exception("DETACH PERMANENTLY is not implemented for dictionaries", ErrorCodes::NOT_IMPLEMENTED); - else - return executeToDictionary(drop.database, drop.table, drop.kind, drop.if_exists, drop.temporary, drop.no_ddl_lock); - } + return executeToTable(drop); else if (!drop.database.empty()) return executeToDatabase(drop); else @@ -122,8 +114,17 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ASTDropQuery & query, DatabaseP if (database && table) { - if (query.as().is_view && !table->isView()) - throw Exception("Table " + table_id.getNameForLogs() + " is not a View", ErrorCodes::LOGICAL_ERROR); + auto & ast_drop_query = query.as(); + + if (ast_drop_query.is_view && !table->isView()) + throw Exception(ErrorCodes::INCORRECT_QUERY, + "Table {} is not a View", + table_id.getNameForLogs()); + + if (ast_drop_query.is_dictionary && !table->isDictionary()) + throw Exception(ErrorCodes::INCORRECT_QUERY, + "Table {} is not a Dictionary", + table_id.getNameForLogs()); /// Now get UUID, so we can wait for table data to be finally dropped table_id.uuid = database->tryGetTableUUID(table_id.table_name); @@ -133,14 +134,24 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ASTDropQuery & query, DatabaseP bool is_replicated_ddl_query = typeid_cast(database.get()) && getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && !is_drop_or_detach_database; + + AccessFlags drop_storage; + + if (table->isView()) + drop_storage = AccessType::DROP_VIEW; + else if (table->isDictionary()) + drop_storage = AccessType::DROP_DICTIONARY; + else + drop_storage = AccessType::DROP_TABLE; + if (is_replicated_ddl_query) { if (query.kind == ASTDropQuery::Kind::Detach) - getContext()->checkAccess(table->isView() ? AccessType::DROP_VIEW : AccessType::DROP_TABLE, table_id); + getContext()->checkAccess(drop_storage, table_id); else if (query.kind == ASTDropQuery::Kind::Truncate) getContext()->checkAccess(AccessType::TRUNCATE, table_id); else if (query.kind == ASTDropQuery::Kind::Drop) - getContext()->checkAccess(table->isView() ? AccessType::DROP_VIEW : AccessType::DROP_TABLE, table_id); + getContext()->checkAccess(drop_storage, table_id); ddl_guard->releaseTableLock(); table.reset(); @@ -149,8 +160,17 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ASTDropQuery & query, DatabaseP if (query.kind == ASTDropQuery::Kind::Detach) { - getContext()->checkAccess(table->isView() ? AccessType::DROP_VIEW : AccessType::DROP_TABLE, table_id); - table->checkTableCanBeDetached(); + getContext()->checkAccess(drop_storage, table_id); + + if (table->isDictionary()) + { + /// If DROP DICTIONARY query is not used, check if Dictionary can be dropped with DROP TABLE query + if (!query.is_dictionary) + table->checkTableCanBeDetached(); + } + else + table->checkTableCanBeDetached(); + table->shutdown(); TableExclusiveLockHolder table_lock; @@ -170,7 +190,11 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ASTDropQuery & query, DatabaseP } else if (query.kind == ASTDropQuery::Kind::Truncate) { + if (table->isDictionary()) + throw Exception("Cannot TRUNCATE dictionary", ErrorCodes::SYNTAX_ERROR); + getContext()->checkAccess(AccessType::TRUNCATE, table_id); + table->checkTableCanBeDropped(); auto table_lock = table->lockExclusively(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); @@ -180,8 +204,16 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ASTDropQuery & query, DatabaseP } else if (query.kind == ASTDropQuery::Kind::Drop) { - getContext()->checkAccess(table->isView() ? AccessType::DROP_VIEW : AccessType::DROP_TABLE, table_id); - table->checkTableCanBeDropped(); + getContext()->checkAccess(drop_storage, table_id); + + if (table->isDictionary()) + { + /// If DROP DICTIONARY query is not used, check if Dictionary can be dropped with DROP TABLE query + if (!query.is_dictionary) + table->checkTableCanBeDropped(); + } + else + table->checkTableCanBeDropped(); table->shutdown(); @@ -199,67 +231,6 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ASTDropQuery & query, DatabaseP return {}; } - -BlockIO InterpreterDropQuery::executeToDictionary( - const String & database_name_, - const String & dictionary_name, - ASTDropQuery::Kind kind, - bool if_exists, - bool is_temporary, - bool no_ddl_lock) -{ - if (is_temporary) - throw Exception("Temporary dictionaries are not possible.", ErrorCodes::SYNTAX_ERROR); - - String database_name = getContext()->resolveDatabase(database_name_); - - auto ddl_guard = (!no_ddl_lock ? DatabaseCatalog::instance().getDDLGuard(database_name, dictionary_name) : nullptr); - query_ptr->as()->database = database_name; - DatabasePtr database = tryGetDatabase(database_name, if_exists); - - bool is_drop_or_detach_database = query_ptr->as()->table.empty(); - bool is_replicated_ddl_query = typeid_cast(database.get()) && - getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && - !is_drop_or_detach_database; - if (is_replicated_ddl_query) - { - if (kind == ASTDropQuery::Kind::Detach) - throw Exception(ErrorCodes::INCORRECT_QUERY, "DETACH DICTIONARY is not allowed for Replicated databases."); - - getContext()->checkAccess(AccessType::DROP_DICTIONARY, database_name, dictionary_name); - - ddl_guard->releaseTableLock(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext()); - } - - if (!database || !database->isDictionaryExist(dictionary_name)) - { - if (!if_exists) - throw Exception( - "Dictionary " + backQuoteIfNeed(database_name) + "." + backQuoteIfNeed(dictionary_name) + " doesn't exist.", - ErrorCodes::UNKNOWN_DICTIONARY); - else - return {}; - } - - if (kind == ASTDropQuery::Kind::Detach) - { - /// Drop dictionary from memory, don't touch data and metadata - getContext()->checkAccess(AccessType::DROP_DICTIONARY, database_name, dictionary_name); - database->detachDictionary(dictionary_name); - } - else if (kind == ASTDropQuery::Kind::Truncate) - { - throw Exception("Cannot TRUNCATE dictionary", ErrorCodes::SYNTAX_ERROR); - } - else if (kind == ASTDropQuery::Kind::Drop) - { - getContext()->checkAccess(AccessType::DROP_DICTIONARY, database_name, dictionary_name); - database->removeDictionary(getContext(), dictionary_name); - } - return {}; -} - BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, ASTDropQuery::Kind kind) { if (kind == ASTDropQuery::Kind::Detach) @@ -351,15 +322,6 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, if (database->shouldBeEmptyOnDetach()) { - /// DETACH or DROP all tables and dictionaries inside database. - /// First we should DETACH or DROP dictionaries because StorageDictionary - /// must be detached only by detaching corresponding dictionary. - for (auto iterator = database->getDictionariesIterator(); iterator->isValid(); iterator->next()) - { - String current_dictionary = iterator->name(); - executeToDictionary(database_name, current_dictionary, query.kind, false, false, false); - } - ASTDropQuery query_for_table; query_for_table.kind = query.kind; query_for_table.if_exists = true; @@ -371,6 +333,7 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr db; UUID table_to_wait = UUIDHelpers::Nil; query_for_table.table = iterator->name(); + query_for_table.is_dictionary = iterator->table()->isDictionary(); executeToTableImpl(query_for_table, db, table_to_wait); uuids_to_wait.push_back(table_to_wait); } diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index e8578a07491..b4a91170bc4 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,8 @@ #include #include +#include + namespace DB { @@ -121,6 +124,7 @@ struct QueryPlanSettings /// Apply query plan optimizations. bool optimize = true; + bool json = false; constexpr static char name[] = "PLAN"; @@ -131,6 +135,7 @@ struct QueryPlanSettings {"actions", query_plan_options.actions}, {"indexes", query_plan_options.indexes}, {"optimize", optimize}, + {"json", json} }; }; @@ -224,6 +229,7 @@ BlockInputStreamPtr InterpreterExplainQuery::executeImpl() MutableColumns res_columns = sample_block.cloneEmptyColumns(); WriteBufferFromOwnString buf; + bool single_line = false; if (ast.getKind() == ASTExplainQuery::ParsedAST) { @@ -256,7 +262,26 @@ BlockInputStreamPtr InterpreterExplainQuery::executeImpl() if (settings.optimize) plan.optimize(QueryPlanOptimizationSettings::fromContext(getContext())); - plan.explainPlan(buf, settings.query_plan_options); + if (settings.json) + { + /// Add extra layers to make plan look more like from postgres. + auto plan_map = std::make_unique(); + plan_map->add("Plan", plan.explainPlan(settings.query_plan_options)); + auto plan_array = std::make_unique(); + plan_array->add(std::move(plan_map)); + + auto format_settings = getFormatSettings(getContext()); + format_settings.json.quote_64bit_integers = false; + + JSONBuilder::FormatSettings json_format_settings{.settings = format_settings}; + JSONBuilder::FormatContext format_context{.out = buf}; + + plan_array->format(json_format_settings, format_context); + + single_line = true; + } + else + plan.explainPlan(buf, settings.query_plan_options); } else if (ast.getKind() == ASTExplainQuery::QueryPipeline) { @@ -289,7 +314,10 @@ BlockInputStreamPtr InterpreterExplainQuery::executeImpl() } } - fillColumn(*res_columns[0], buf.str()); + if (single_line) + res_columns[0]->insertData(buf.str().data(), buf.str().size()); + else + fillColumn(*res_columns[0], buf.str()); return std::make_shared(sample_block.cloneWithColumns(std::move(res_columns))); } diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index 1a247dbf6e6..967d3e7f570 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -45,17 +45,33 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() ASTPtr create_query; ASTQueryWithTableAndOutput * show_query; if ((show_query = query_ptr->as()) || - (show_query = query_ptr->as())) + (show_query = query_ptr->as()) || + (show_query = query_ptr->as())) { auto resolve_table_type = show_query->temporary ? Context::ResolveExternal : Context::ResolveOrdinary; auto table_id = getContext()->resolveStorageID(*show_query, resolve_table_type); - getContext()->checkAccess(AccessType::SHOW_COLUMNS, table_id); + + bool is_dictionary = static_cast(query_ptr->as()); + + if (is_dictionary) + getContext()->checkAccess(AccessType::SHOW_DICTIONARIES, table_id); + else + getContext()->checkAccess(AccessType::SHOW_COLUMNS, table_id); + create_query = DatabaseCatalog::instance().getDatabase(table_id.database_name)->getCreateTableQuery(table_id.table_name, getContext()); + + auto & ast_create_query = create_query->as(); if (query_ptr->as()) { - auto & ast_create_query = create_query->as(); if (!ast_create_query.isView()) - throw Exception(backQuote(ast_create_query.database) + "." + backQuote(ast_create_query.table) + " is not a VIEW", ErrorCodes::BAD_ARGUMENTS); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}.{} is not a VIEW", + backQuote(ast_create_query.database), backQuote(ast_create_query.table)); + } + else if (is_dictionary) + { + if (!ast_create_query.is_dictionary) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}.{} is not a DICTIONARY", + backQuote(ast_create_query.database), backQuote(ast_create_query.table)); } } else if ((show_query = query_ptr->as())) @@ -66,14 +82,6 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() getContext()->checkAccess(AccessType::SHOW_DATABASES, show_query->database); create_query = DatabaseCatalog::instance().getDatabase(show_query->database)->getCreateDatabaseQuery(); } - else if ((show_query = query_ptr->as())) - { - if (show_query->temporary) - throw Exception("Temporary dictionaries are not possible.", ErrorCodes::SYNTAX_ERROR); - show_query->database = getContext()->resolveDatabase(show_query->database); - getContext()->checkAccess(AccessType::SHOW_DICTIONARIES, show_query->database, show_query->table); - create_query = DatabaseCatalog::instance().getDatabase(show_query->database)->getCreateDictionaryQuery(show_query->table); - } if (!create_query) throw Exception("Unable to show the create query of " + show_query->table + ". Maybe it was created by the system.", ErrorCodes::THERE_IS_NO_QUERY); diff --git a/src/Interpreters/examples/CMakeLists.txt b/src/Interpreters/examples/CMakeLists.txt new file mode 100644 index 00000000000..6916c255b36 --- /dev/null +++ b/src/Interpreters/examples/CMakeLists.txt @@ -0,0 +1,37 @@ +add_executable (hash_map hash_map.cpp) +target_include_directories (hash_map SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) +target_link_libraries (hash_map PRIVATE dbms) + +add_executable (hash_map_lookup hash_map_lookup.cpp) +target_include_directories (hash_map_lookup SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) +target_link_libraries (hash_map_lookup PRIVATE dbms) + +add_executable (hash_map3 hash_map3.cpp) +target_link_libraries (hash_map3 PRIVATE dbms ${FARMHASH_LIBRARIES} metrohash) + +add_executable (hash_map_string hash_map_string.cpp) +target_include_directories (hash_map_string SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) +target_link_libraries (hash_map_string PRIVATE dbms) + +add_executable (hash_map_string_2 hash_map_string_2.cpp) +target_link_libraries (hash_map_string_2 PRIVATE dbms) + +add_executable (hash_map_string_3 hash_map_string_3.cpp) +target_link_libraries (hash_map_string_3 PRIVATE dbms ${FARMHASH_LIBRARIES} metrohash) + +add_executable (hash_map_string_small hash_map_string_small.cpp) +target_include_directories (hash_map_string_small SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) +target_link_libraries (hash_map_string_small PRIVATE dbms) + +add_executable (string_hash_map string_hash_map.cpp) +target_link_libraries (string_hash_map PRIVATE dbms) + +add_executable (string_hash_map_aggregation string_hash_map.cpp) +target_link_libraries (string_hash_map_aggregation PRIVATE dbms) + +add_executable (string_hash_set string_hash_set.cpp) +target_link_libraries (string_hash_set PRIVATE dbms) + +add_executable (two_level_hash_map two_level_hash_map.cpp) +target_include_directories (two_level_hash_map SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) +target_link_libraries (two_level_hash_map PRIVATE dbms) diff --git a/src/Interpreters/tests/hash_map.cpp b/src/Interpreters/examples/hash_map.cpp similarity index 100% rename from src/Interpreters/tests/hash_map.cpp rename to src/Interpreters/examples/hash_map.cpp diff --git a/src/Interpreters/tests/hash_map3.cpp b/src/Interpreters/examples/hash_map3.cpp similarity index 100% rename from src/Interpreters/tests/hash_map3.cpp rename to src/Interpreters/examples/hash_map3.cpp diff --git a/src/Interpreters/tests/hash_map_lookup.cpp b/src/Interpreters/examples/hash_map_lookup.cpp similarity index 100% rename from src/Interpreters/tests/hash_map_lookup.cpp rename to src/Interpreters/examples/hash_map_lookup.cpp diff --git a/src/Interpreters/tests/hash_map_string.cpp b/src/Interpreters/examples/hash_map_string.cpp similarity index 100% rename from src/Interpreters/tests/hash_map_string.cpp rename to src/Interpreters/examples/hash_map_string.cpp diff --git a/src/Interpreters/tests/hash_map_string_2.cpp b/src/Interpreters/examples/hash_map_string_2.cpp similarity index 100% rename from src/Interpreters/tests/hash_map_string_2.cpp rename to src/Interpreters/examples/hash_map_string_2.cpp diff --git a/src/Interpreters/tests/hash_map_string_3.cpp b/src/Interpreters/examples/hash_map_string_3.cpp similarity index 100% rename from src/Interpreters/tests/hash_map_string_3.cpp rename to src/Interpreters/examples/hash_map_string_3.cpp diff --git a/src/Interpreters/tests/hash_map_string_small.cpp b/src/Interpreters/examples/hash_map_string_small.cpp similarity index 100% rename from src/Interpreters/tests/hash_map_string_small.cpp rename to src/Interpreters/examples/hash_map_string_small.cpp diff --git a/src/Interpreters/tests/string_hash_map.cpp b/src/Interpreters/examples/string_hash_map.cpp similarity index 100% rename from src/Interpreters/tests/string_hash_map.cpp rename to src/Interpreters/examples/string_hash_map.cpp diff --git a/src/Interpreters/tests/string_hash_set.cpp b/src/Interpreters/examples/string_hash_set.cpp similarity index 100% rename from src/Interpreters/tests/string_hash_set.cpp rename to src/Interpreters/examples/string_hash_set.cpp diff --git a/src/Interpreters/tests/two_level_hash_map.cpp b/src/Interpreters/examples/two_level_hash_map.cpp similarity index 100% rename from src/Interpreters/tests/two_level_hash_map.cpp rename to src/Interpreters/examples/two_level_hash_map.cpp diff --git a/src/Interpreters/tests/CMakeLists.txt b/src/Interpreters/tests/CMakeLists.txt index 8905d2fe6e6..e69de29bb2d 100644 --- a/src/Interpreters/tests/CMakeLists.txt +++ b/src/Interpreters/tests/CMakeLists.txt @@ -1,45 +0,0 @@ -add_executable (hash_map hash_map.cpp) -target_include_directories (hash_map SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map PRIVATE dbms) - -add_executable (hash_map_lookup hash_map_lookup.cpp) -target_include_directories (hash_map_lookup SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map_lookup PRIVATE dbms) - -add_executable (hash_map3 hash_map3.cpp) -target_link_libraries (hash_map3 PRIVATE dbms ${FARMHASH_LIBRARIES} metrohash) - -add_executable (hash_map_string hash_map_string.cpp) -target_include_directories (hash_map_string SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map_string PRIVATE dbms) - -add_executable (hash_map_string_2 hash_map_string_2.cpp) -target_link_libraries (hash_map_string_2 PRIVATE dbms) - -add_executable (hash_map_string_3 hash_map_string_3.cpp) -target_link_libraries (hash_map_string_3 PRIVATE dbms ${FARMHASH_LIBRARIES} metrohash) - -add_executable (hash_map_string_small hash_map_string_small.cpp) -target_include_directories (hash_map_string_small SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) -target_link_libraries (hash_map_string_small PRIVATE dbms) - -add_executable (string_hash_map string_hash_map.cpp) -target_link_libraries (string_hash_map PRIVATE dbms) - -add_executable (string_hash_map_aggregation string_hash_map.cpp) -target_link_libraries (string_hash_map_aggregation PRIVATE dbms) - -add_executable (string_hash_set string_hash_set.cpp) -target_link_libraries (string_hash_set PRIVATE dbms) - -add_executable (two_level_hash_map two_level_hash_map.cpp) -target_include_directories (two_level_hash_map SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) -target_link_libraries (two_level_hash_map PRIVATE dbms) - -add_executable (in_join_subqueries_preprocessor in_join_subqueries_preprocessor.cpp) -target_link_libraries (in_join_subqueries_preprocessor PRIVATE clickhouse_aggregate_functions dbms clickhouse_parsers) - -if (OS_LINUX) - add_executable (internal_iotop internal_iotop.cpp) - target_link_libraries (internal_iotop PRIVATE dbms) -endif () diff --git a/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp b/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp deleted file mode 100644 index efe3882d086..00000000000 --- a/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp +++ /dev/null @@ -1,1264 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - - -namespace DB -{ - namespace ErrorCodes - { - extern const int DISTRIBUTED_IN_JOIN_SUBQUERY_DENIED; - } -} - - -/// Simplified version of the StorageDistributed class. -class StorageDistributedFake final : public ext::shared_ptr_helper, public DB::IStorage -{ - friend struct ext::shared_ptr_helper; -public: - std::string getName() const override { return "DistributedFake"; } - bool isRemote() const override { return true; } - size_t getShardCount() const { return shard_count; } - std::string getRemoteDatabaseName() const { return remote_database; } - std::string getRemoteTableName() const { return remote_table; } - -protected: - StorageDistributedFake(const std::string & remote_database_, const std::string & remote_table_, size_t shard_count_) - : IStorage({"", ""}), remote_database(remote_database_), remote_table(remote_table_), shard_count(shard_count_) - { - } - -private: - const std::string remote_database; - const std::string remote_table; - size_t shard_count; -}; - - -class CheckShardsAndTablesMock : public DB::InJoinSubqueriesPreprocessor::CheckShardsAndTables -{ -public: - bool hasAtLeastTwoShards(const DB::IStorage & table) const override - { - if (!table.isRemote()) - return false; - - const StorageDistributedFake * distributed = dynamic_cast(&table); - if (!distributed) - return false; - - return distributed->getShardCount() >= 2; - } - - std::pair - getRemoteDatabaseAndTableName(const DB::IStorage & table) const override - { - const StorageDistributedFake & distributed = dynamic_cast(table); - return { distributed.getRemoteDatabaseName(), distributed.getRemoteTableName() }; - } -}; - - -struct TestEntry -{ - unsigned int line_num; - std::string input; - std::string expected_output; - size_t shard_count; - DB::DistributedProductMode mode; - bool expected_success; -}; - -using TestEntries = std::vector; -using TestResult = std::pair; - -TestResult check(const TestEntry & entry); -bool parse(DB::ASTPtr & ast, const std::string & query); -bool equals(const DB::ASTPtr & lhs, const DB::ASTPtr & rhs); -void reorder(DB::IAST * ast); - - -TestEntries entries = -{ - /// Trivial query. - - { - __LINE__, - "SELECT 1", - "SELECT 1", - 0, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT 1", - "SELECT 1", - 1, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT 1", - "SELECT 1", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT 1", - "SELECT 1", - 0, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT 1", - "SELECT 1", - 1, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT 1", - "SELECT 1", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT 1", - "SELECT 1", - 0, - DB::DistributedProductMode::LOCAL, - true - }, - - { - __LINE__, - "SELECT 1", - "SELECT 1", - 1, - DB::DistributedProductMode::LOCAL, - true - }, - - { - __LINE__, - "SELECT 1", - "SELECT 1", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - { - __LINE__, - "SELECT 1", - "SELECT 1", - 0, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT 1", - "SELECT 1", - 1, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT 1", - "SELECT 1", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - /// Section IN / depth 1 - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM test.visits_all)", - 1, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM test.visits_all)", - 1, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::DENY, - false - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote_db.remote_visits)", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote_db.remote_visits)", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote_db.remote_visits)", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote_db.remote_visits)", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote_db.remote_visits)", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section NOT IN / depth 1 - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID NOT IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID NOT IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID NOT IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID NOT IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::DENY, - false - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID NOT IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL NOT IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID NOT IN (SELECT UserID FROM remote_db.remote_visits)", - "SELECT count() FROM test.visits_all WHERE UserID NOT IN (SELECT UserID FROM remote_db.remote_visits)", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID NOT IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID NOT IN (SELECT UserID FROM remote_db.remote_visits)", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID NOT IN (SELECT UserID FROM remote_db.remote_visits)", - "SELECT count() FROM test.visits_all WHERE UserID NOT IN (SELECT UserID FROM remote_db.remote_visits)", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section GLOBAL IN / depth 1 - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section GLOBAL NOT IN / depth 1 - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL NOT IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL NOT IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL NOT IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL NOT IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL NOT IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL NOT IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL NOT IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL NOT IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section JOIN / depth 1 - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - 2, - DB::DistributedProductMode::DENY, - false - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM remote_db.remote_visits) USING UserID", - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM remote_db.remote_visits) USING UserID", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM remote_db.remote_visits) USING UserID", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM remote_db.remote_visits) USING UserID", - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM remote_db.remote_visits) USING UserID", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section GLOBAL JOIN / depth 1 - - { - __LINE__, - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM test.visits_all) USING UserID", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section JOIN / depth 1 / 2 of the subquery. - - { - __LINE__, - "SELECT UserID FROM (SELECT UserID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE RegionID = 2) USING UserID", - "SELECT UserID FROM (SELECT UserID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE RegionID = 2) USING UserID", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT UserID FROM (SELECT UserID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT UserID FROM remote_db.remote_visits WHERE RegionID = 2) USING UserID", - "SELECT UserID FROM (SELECT UserID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT UserID FROM remote_db.remote_visits WHERE RegionID = 2) USING UserID", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT UserID FROM (SELECT UserID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE RegionID = 2) USING UserID", - "SELECT UserID FROM (SELECT UserID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE RegionID = 2) USING UserID", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT UserID FROM (SELECT UserID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE RegionID = 2) USING UserID", - "SELECT UserID FROM (SELECT UserID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE RegionID = 2) USING UserID", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT UserID FROM (SELECT UserID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE RegionID = 2) USING UserID", - "SELECT UserID FROM (SELECT UserID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE RegionID = 2) USING UserID", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section IN / depth 1 / table at level 2 - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - 2, - DB::DistributedProductMode::DENY, - false - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - "SELECT count() FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM (SELECT UserID FROM remote_db.remote_visits))", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM (SELECT UserID FROM remote_db.remote_visits))", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM (SELECT UserID FROM remote_db.remote_visits))", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM (SELECT UserID FROM remote_db.remote_visits))", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM (SELECT UserID FROM remote_db.remote_visits))", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section GLOBAL IN / depth 1 / table at level 2 - - { - __LINE__, - "SELECT UserID FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - "SELECT UserID FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - "SELECT UserID FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - "SELECT UserID FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - "SELECT UserID FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all))", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section IN at level 1, GLOBAL IN section at level 2. - - { - __LINE__, - "SELECT UserID FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote_db.remote_visits WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all)))", - "SELECT UserID FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote_db.remote_visits WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all)))", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote_db.remote_visits WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all)))", - "SELECT UserID FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote_db.remote_visits WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all)))", - 2, - DB::DistributedProductMode::DENY, - false - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote_db.remote_visits WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM test.visits_all)))", - "SELECT UserID FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote_db.remote_visits WHERE UserID GLOBAL IN (SELECT UserID FROM (SELECT UserID FROM remote_db.remote_visits)))", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section JOIN / depth 1 / table at level 2 - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM (SELECT UserID FROM test.visits_all)) USING UserID", - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM (SELECT UserID FROM test.visits_all)) USING UserID", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM (SELECT UserID FROM test.visits_all)) USING UserID", - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM (SELECT UserID FROM test.visits_all)) USING UserID", - 2, - DB::DistributedProductMode::DENY, - false - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM (SELECT UserID FROM test.visits_all)) USING UserID", - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM (SELECT UserID FROM test.visits_all)) USING UserID", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM (SELECT UserID FROM remote_db.remote_visits)) USING UserID", - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM (SELECT UserID FROM remote_db.remote_visits)) USING UserID", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM (SELECT UserID FROM test.visits_all)) USING UserID", - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM (SELECT UserID FROM remote_db.remote_visits)) USING UserID", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM (SELECT UserID FROM remote_db.remote_visits)) USING UserID", - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM (SELECT UserID FROM remote_db.remote_visits)) USING UserID", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section IN / depth 2 - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM test.visits_all WHERE OtherID = 1))", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM test.visits_all WHERE OtherID = 1))", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID GLOBAL IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM remote_db.remote_visits WHERE OtherID = 1))", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID GLOBAL IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM remote_db.remote_visits WHERE OtherID = 1))", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM test.visits_all WHERE OtherID = 1))", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM test.visits_all WHERE OtherID = 1))", - 2, - DB::DistributedProductMode::DENY, - false - }, - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID GLOBAL IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM test.visits_all WHERE OtherID = 1))", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID GLOBAL IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM test.visits_all WHERE OtherID = 1))", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM remote_db.remote_visits WHERE OtherID = 1))", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID GLOBAL IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM remote_db.remote_visits WHERE OtherID = 1))", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM test.visits_all WHERE OtherID = 1))", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM remote_db.remote_visits WHERE BrowserID IN (SELECT BrowserID FROM remote_db.remote_visits WHERE OtherID = 1))", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID GLOBAL IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM test.visits_all WHERE OtherID = 1))", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID GLOBAL IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM test.visits_all WHERE OtherID = 1))", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM test.visits_all WHERE BrowserID IN (SELECT BrowserID FROM remote_db.remote_visits WHERE OtherID = 1))", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM remote_db.remote_visits WHERE BrowserID IN (SELECT BrowserID FROM remote_db.remote_visits WHERE OtherID = 1))", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section JOIN / depth 2 - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM test.visits_all ALL INNER JOIN (SELECT CounterID FROM (SELECT CounterID FROM test.visits_all)) USING CounterID)", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM test.visits_all ALL INNER JOIN (SELECT CounterID FROM (SELECT CounterID FROM test.visits_all)) USING CounterID)", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID GLOBAL IN (SELECT CounterID FROM test.visits_all ALL INNER JOIN (SELECT CounterID FROM (SELECT CounterID FROM remote_db.remote_visits)) USING CounterID)", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID GLOBAL IN (SELECT CounterID FROM test.visits_all ALL INNER JOIN (SELECT CounterID FROM (SELECT CounterID FROM remote_db.remote_visits)) USING CounterID)", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID GLOBAL IN (SELECT CounterID FROM test.visits_all ALL INNER JOIN (SELECT CounterID FROM (SELECT CounterID FROM test.visits_all)) USING CounterID)", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID GLOBAL IN (SELECT CounterID FROM test.visits_all ALL INNER JOIN (SELECT CounterID FROM (SELECT CounterID FROM test.visits_all)) USING CounterID)", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM test.visits_all ALL INNER JOIN (SELECT CounterID FROM (SELECT CounterID FROM test.visits_all)) USING CounterID)", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID GLOBAL IN (SELECT CounterID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT CounterID FROM (SELECT CounterID FROM test.visits_all)) USING CounterID)", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM test.visits_all ALL INNER JOIN (SELECT CounterID FROM (SELECT CounterID FROM test.visits_all)) USING CounterID)", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM remote_db.remote_visits ALL INNER JOIN (SELECT CounterID FROM (SELECT CounterID FROM remote_db.remote_visits)) USING CounterID)", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section JOIN / depth 2 - - { - __LINE__, - "SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM (SELECT OtherID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2) USING OtherID)", - "SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM (SELECT OtherID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2) USING OtherID)", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all WHERE OtherID GLOBAL IN (SELECT OtherID FROM (SELECT OtherID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT OtherID FROM remote_db.remote_visits WHERE RegionID = 2) USING OtherID)", - "SELECT UserID FROM test.visits_all WHERE OtherID GLOBAL IN (SELECT OtherID FROM (SELECT OtherID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT OtherID FROM remote_db.remote_visits WHERE RegionID = 2) USING OtherID)", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all WHERE OtherID GLOBAL IN (SELECT OtherID FROM (SELECT OtherID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2) USING OtherID)", - "SELECT UserID FROM test.visits_all WHERE OtherID GLOBAL IN (SELECT OtherID FROM (SELECT OtherID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2) USING OtherID)", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM (SELECT OtherID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2) USING OtherID)", - "SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM (SELECT OtherID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2) USING OtherID)", - 2, - DB::DistributedProductMode::DENY, - false - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM (SELECT OtherID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2) USING OtherID)", - "SELECT UserID FROM test.visits_all WHERE OtherID GLOBAL IN (SELECT OtherID FROM (SELECT OtherID FROM test.visits_all WHERE RegionID = 1) GLOBAL ALL INNER JOIN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2) USING OtherID)", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM (SELECT OtherID FROM test.visits_all WHERE RegionID = 1) ALL INNER JOIN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2) USING OtherID)", - "SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM (SELECT OtherID FROM remote_db.remote_visits WHERE RegionID = 1) ALL INNER JOIN (SELECT OtherID FROM remote_db.remote_visits WHERE RegionID = 2) USING OtherID)", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section JOIN / section IN - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2)) USING UserID", - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2)) USING UserID", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2)) USING UserID", - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2)) USING UserID", - 2, - DB::DistributedProductMode::DENY, - false - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2)) USING UserID", - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2)) USING UserID", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM remote_db.remote_visits WHERE RegionID = 2)) USING UserID", - "SELECT UserID FROM test.visits_all GLOBAL ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM remote_db.remote_visits WHERE RegionID = 2)) USING UserID", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM test.visits_all WHERE OtherID IN (SELECT OtherID FROM test.visits_all WHERE RegionID = 2)) USING UserID", - "SELECT UserID FROM test.visits_all ALL INNER JOIN (SELECT UserID FROM remote_db.remote_visits WHERE OtherID IN (SELECT OtherID FROM remote_db.remote_visits WHERE RegionID = 2)) USING UserID", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Table function. - - { - __LINE__, - "SELECT count() FROM remote('127.0.0.{1,2}', test, visits_all) WHERE UserID IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM remote('127.0.0.{1,2}', test, visits_all) WHERE UserID IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT count() FROM remote('127.0.0.{1,2}', test, visits_all) WHERE UserID IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM remote('127.0.0.{1,2}', test, visits_all) WHERE UserID IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote('127.0.0.{1,2}', test, visits_all))", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote('127.0.0.{1,2}', test, visits_all))", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT count() FROM remote('127.0.0.{1,2}', test, visits_all) WHERE UserID IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM remote('127.0.0.{1,2}', test, visits_all) WHERE UserID IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote('127.0.0.{1,2}', test, visits_all))", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote('127.0.0.{1,2}', test, visits_all))", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT count() FROM remote('127.0.0.{1,2}', test, visits_all) WHERE UserID IN (SELECT UserID FROM test.visits_all)", - "SELECT count() FROM remote('127.0.0.{1,2}', test, visits_all) WHERE UserID IN (SELECT UserID FROM test.visits_all)", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote('127.0.0.{1,2}', test, visits_all))", - "SELECT count() FROM test.visits_all WHERE UserID IN (SELECT UserID FROM remote('127.0.0.{1,2}', test, visits_all))", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Section IN / depth 2 / two distributed tables - - { - __LINE__, - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM test.hits_all WHERE BrowserID IN (SELECT BrowserID FROM test.visits_all WHERE OtherID = 1))", - "SELECT UserID, RegionID FROM test.visits_all WHERE CounterID IN (SELECT CounterID FROM distant_db.distant_hits WHERE BrowserID IN (SELECT BrowserID FROM remote_db.remote_visits WHERE OtherID = 1))", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Aggregate function. - - { - __LINE__, - "SELECT sum(RegionID IN (SELECT RegionID from test.hits_all)) FROM test.visits_all", - "SELECT sum(RegionID IN (SELECT RegionID from test.hits_all)) FROM test.visits_all", - 2, - DB::DistributedProductMode::ALLOW, - true - }, - - { - __LINE__, - "SELECT sum(RegionID IN (SELECT RegionID from test.hits_all)) FROM test.visits_all", - "SELECT sum(RegionID IN (SELECT RegionID from test.hits_all)) FROM test.visits_all", - 2, - DB::DistributedProductMode::DENY, - false - }, - - { - __LINE__, - "SELECT sum(RegionID GLOBAL IN (SELECT RegionID from test.hits_all)) FROM test.visits_all", - "SELECT sum(RegionID GLOBAL IN (SELECT RegionID from test.hits_all)) FROM test.visits_all", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT sum(RegionID IN (SELECT RegionID from test.hits_all)) FROM test.visits_all", - "SELECT sum(RegionID GLOBAL IN (SELECT RegionID from test.hits_all)) FROM test.visits_all", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT sum(RegionID GLOBAL IN (SELECT RegionID from test.hits_all)) FROM test.visits_all", - "SELECT sum(RegionID GLOBAL IN (SELECT RegionID from test.hits_all)) FROM test.visits_all", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - { - __LINE__, - "SELECT sum(RegionID IN (SELECT RegionID from test.hits_all)) FROM test.visits_all", - "SELECT sum(RegionID IN (SELECT RegionID from distant_db.distant_hits)) FROM test.visits_all", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - /// Miscellaneous. - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE x GLOBAL IN (SELECT x FROM test.visits_all WHERE x GLOBAL IN (SELECT x FROM test.visits_all))", - "SELECT count() FROM test.visits_all WHERE x GLOBAL IN (SELECT x FROM test.visits_all WHERE x GLOBAL IN (SELECT x FROM test.visits_all))", - 2, - DB::DistributedProductMode::DENY, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE x GLOBAL IN (SELECT x FROM test.visits_all WHERE x GLOBAL IN (SELECT x FROM test.visits_all))", - "SELECT count() FROM test.visits_all WHERE x GLOBAL IN (SELECT x FROM test.visits_all WHERE x GLOBAL IN (SELECT x FROM test.visits_all))", - 2, - DB::DistributedProductMode::LOCAL, - true - }, - - { - __LINE__, - "SELECT count() FROM test.visits_all WHERE x GLOBAL IN (SELECT x FROM test.visits_all WHERE x GLOBAL IN (SELECT x FROM test.visits_all))", - "SELECT count() FROM test.visits_all WHERE x GLOBAL IN (SELECT x FROM test.visits_all WHERE x GLOBAL IN (SELECT x FROM test.visits_all))", - 2, - DB::DistributedProductMode::GLOBAL, - true - }, - - { - __LINE__, - "SELECT UserID FROM (SELECT UserID FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM test.hits_all))", - "SELECT UserID FROM (SELECT UserID FROM test.visits_all WHERE UserID GLOBAL IN (SELECT UserID FROM test.hits_all))", - 2, - DB::DistributedProductMode::DENY, - true - } -}; - - -static bool run() -{ - unsigned int count = 0; - unsigned int i = 1; - - for (const auto & entry : entries) - { - auto res = check(entry); - if (res.first) - { - ++count; - } - else - std::cout << "Test " << i << " at line " << entry.line_num << " failed.\n" - "Expected: " << entry.expected_output << ".\n" - "Received: " << res.second << "\n"; - - ++i; - } - - std::cout << count << " out of " << entries.size() << " test(s) passed.\n"; - - return count == entries.size(); -} - - -TestResult check(const TestEntry & entry) -{ - static auto shared_context = DB::Context::createShared(); - static auto context = DB::Context::createGlobal(shared_context.get()); - context->makeGlobalContext(); - - try - { - - auto storage_distributed_visits = StorageDistributedFake::create("remote_db", "remote_visits", entry.shard_count); - auto storage_distributed_hits = StorageDistributedFake::create("distant_db", "distant_hits", entry.shard_count); - - DB::DatabasePtr database = std::make_shared("test", "./metadata/test/", context); - DB::DatabaseCatalog::instance().attachDatabase("test", database); - database->attachTable("visits_all", storage_distributed_visits); - database->attachTable("hits_all", storage_distributed_hits); - context->setCurrentDatabase("test"); - context->setSetting("distributed_product_mode", entry.mode); - - /// Parse and process the incoming query. - DB::ASTPtr ast_input; - if (!parse(ast_input, entry.input)) - return TestResult(false, "parse error"); - - bool success = true; - - try - { - DB::InJoinSubqueriesPreprocessor::SubqueryTables renamed; - DB::InJoinSubqueriesPreprocessor(context, renamed, std::make_unique()).visit(ast_input); - } - catch (const DB::Exception & ex) - { - if (ex.code() == DB::ErrorCodes::DISTRIBUTED_IN_JOIN_SUBQUERY_DENIED) - success = false; - else - throw; - } - catch (...) - { - throw; - } - - if (success != entry.expected_success) - return TestResult(false, "unexpected result"); - - /// Parse the expected result. - DB::ASTPtr ast_expected; - if (!parse(ast_expected, entry.expected_output)) - return TestResult(false, "parse error"); - - /// Compare the processed query and the expected result. - bool res = equals(ast_input, ast_expected); - std::string output = DB::queryToString(ast_input); - - DB::DatabaseCatalog::instance().detachDatabase("test"); - return TestResult(res, output); - } - catch (DB::Exception & e) - { - DB::DatabaseCatalog::instance().detachDatabase("test"); - return TestResult(false, e.displayText()); - } -} - -bool parse(DB::ASTPtr & ast, const std::string & query) -{ - DB::ParserSelectQuery parser; - std::string message; - const auto * begin = query.data(); - const auto * end = begin + query.size(); - ast = DB::tryParseQuery(parser, begin, end, message, false, "", false, 0, 0); - return ast != nullptr; -} - -bool equals(const DB::ASTPtr & lhs, const DB::ASTPtr & rhs) -{ - DB::ASTPtr lhs_reordered = lhs->clone(); - reorder(&*lhs_reordered); - - DB::ASTPtr rhs_reordered = rhs->clone(); - reorder(&*rhs_reordered); - - return lhs_reordered->getTreeHash() == rhs_reordered->getTreeHash(); -} - -void reorder(DB::IAST * ast) -{ - if (ast == nullptr) - return; - - auto & children = ast->children; - if (children.empty()) - return; - - for (auto & child : children) - reorder(&*child); - - std::sort(children.begin(), children.end(), [](const DB::ASTPtr & lhs, const DB::ASTPtr & rhs) - { - return lhs->getTreeHash() < rhs->getTreeHash(); - }); -} - -int main() -{ - return run() ? EXIT_SUCCESS : EXIT_FAILURE; -} diff --git a/src/Interpreters/tests/internal_iotop.cpp b/src/Interpreters/tests/internal_iotop.cpp deleted file mode 100644 index 6025a46e9b7..00000000000 --- a/src/Interpreters/tests/internal_iotop.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -std::mutex mutex; - - -static std::ostream & operator << (std::ostream & stream, const ::taskstats & stat) -{ -#define PRINT(field) (stream << #field << " " << stat.field) - - PRINT(ac_pid) << ", "; - - PRINT(read_bytes) << ", "; - PRINT(write_bytes) << ", "; - - PRINT(read_char) << ", "; - PRINT(write_char) << ", "; - - PRINT(swapin_delay_total) << ", "; - PRINT(blkio_delay_total) << ", "; - PRINT(cpu_delay_total) << ", "; - - PRINT(ac_pid) << ", "; - - PRINT(ac_utime) << ", "; - PRINT(ac_stime) << ", "; - -#undef PRINT - - return stream; -} - -using namespace DB; - - -static void do_io(size_t id) -{ - ::taskstats stat; - int tid = getThreadId(); - TaskStatsInfoGetter get_info; - - get_info.getStat(stat, tid); - { - std::lock_guard lock(mutex); - std::cerr << "#" << id << ", tid " << tid << ", initial\n" << stat << "\n"; - } - - size_t copy_size = 1048576 * (1 + id); - std::string path_dst = "test_out_" + std::to_string(id); - - { - size_t page_size = static_cast(::getPageSize()); - ReadBufferFromFile rb("/dev/urandom"); - WriteBufferFromFile wb(path_dst, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_CREAT | O_TRUNC | O_DIRECT, 0666, nullptr, page_size); - copyData(rb, wb, copy_size); - wb.close(); - } - - get_info.getStat(stat, tid); - { - std::lock_guard lock(mutex); - std::cerr << "#" << id << ", tid " << tid << ", step1\n" << stat << "\n"; - } - - { - ReadBufferFromFile rb(path_dst); - WriteBufferFromOwnString wb; - copyData(rb, wb, copy_size); - } - - get_info.getStat(stat, tid); - { - std::lock_guard lock(mutex); - std::cerr << "#" << id << ", tid " << tid << ", step2\n" << stat << "\n"; - } - - { - ReadBufferFromFile rb(path_dst); - WriteBufferFromOwnString wb; - copyData(rb, wb, copy_size); - } - - get_info.getStat(stat, tid); - { - std::lock_guard lock(mutex); - std::cerr << "#" << id << ", tid " << tid << ", step3\n" << stat << "\n"; - } - - Poco::File(path_dst).remove(false); -} - -static void test_perf() -{ - - ::taskstats stat; - int tid = getThreadId(); - TaskStatsInfoGetter get_info; - - rusage rusage; - - constexpr size_t num_samples = 1000000; - { - Stopwatch watch; - for (size_t i = 0; i < num_samples; ++i) - getrusage(RUSAGE_THREAD, &rusage); - - auto ms = watch.elapsedMilliseconds(); - if (ms > 0) - std::cerr << "RUsage: " << double(ms) / num_samples << " ms per call, " << 1000 * num_samples / ms << " calls per second\n"; - } - - { - Stopwatch watch; - for (size_t i = 0; i < num_samples; ++i) - get_info.getStat(stat, tid); - - auto ms = watch.elapsedMilliseconds(); - if (ms > 0) - std::cerr << "Netlink: " << double(ms) / num_samples << " ms per call, " << 1000 * num_samples / ms << " calls per second\n"; - } - - std::cerr << stat << "\n"; -} - -int main() -try -{ - std::cerr << "pid " << getpid() << "\n"; - - size_t num_threads = 2; - ThreadPool pool(num_threads); - for (size_t i = 0; i < num_threads; ++i) - pool.scheduleOrThrowOnError([i]() { do_io(i); }); - pool.wait(); - - test_perf(); - return 0; -} -catch (...) -{ - std::cerr << getCurrentExceptionMessage(true); - return -1; -} - diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 90998077a5a..105e1e11365 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -54,7 +54,7 @@ SRCS( ExpressionAnalyzer.cpp ExternalDictionariesLoader.cpp ExternalLoader.cpp - ExternalLoaderDatabaseConfigRepository.cpp + ExternalLoaderDictionaryStorageConfigRepository.cpp ExternalLoaderTempConfigRepository.cpp ExternalLoaderXMLConfigRepository.cpp ExternalModelsLoader.cpp diff --git a/src/Parsers/CMakeLists.txt b/src/Parsers/CMakeLists.txt index 167d7b155c6..5aaa5c32f92 100644 --- a/src/Parsers/CMakeLists.txt +++ b/src/Parsers/CMakeLists.txt @@ -9,6 +9,6 @@ if (USE_DEBUG_HELPERS) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}") endif () -if(ENABLE_TESTS) - add_subdirectory(tests) +if(ENABLE_EXAMPLES) + add_subdirectory(examples) endif() diff --git a/src/Parsers/ParserRenameQuery.cpp b/src/Parsers/ParserRenameQuery.cpp index 7fa4e6e5408..e3b35249cd6 100644 --- a/src/Parsers/ParserRenameQuery.cpp +++ b/src/Parsers/ParserRenameQuery.cpp @@ -42,6 +42,7 @@ bool ParserRenameQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_rename_table("RENAME TABLE"); ParserKeyword s_exchange_tables("EXCHANGE TABLES"); ParserKeyword s_rename_dictionary("RENAME DICTIONARY"); + ParserKeyword s_exchange_dictionaries("EXCHANGE DICTIONARIES"); ParserKeyword s_rename_database("RENAME DATABASE"); ParserKeyword s_to("TO"); ParserKeyword s_and("AND"); @@ -56,6 +57,11 @@ bool ParserRenameQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) exchange = true; else if (s_rename_dictionary.ignore(pos, expected)) dictionary = true; + else if (s_exchange_dictionaries.ignore(pos, expected)) + { + exchange = true; + dictionary = true; + } else if (s_rename_database.ignore(pos, expected)) { ASTPtr from_db; diff --git a/src/Parsers/tests/CMakeLists.txt b/src/Parsers/examples/CMakeLists.txt similarity index 100% rename from src/Parsers/tests/CMakeLists.txt rename to src/Parsers/examples/CMakeLists.txt diff --git a/src/Parsers/tests/create_parser.cpp b/src/Parsers/examples/create_parser.cpp similarity index 100% rename from src/Parsers/tests/create_parser.cpp rename to src/Parsers/examples/create_parser.cpp diff --git a/src/Parsers/tests/create_parser_fuzzer.cpp b/src/Parsers/examples/create_parser_fuzzer.cpp similarity index 100% rename from src/Parsers/tests/create_parser_fuzzer.cpp rename to src/Parsers/examples/create_parser_fuzzer.cpp diff --git a/src/Parsers/tests/lexer.cpp b/src/Parsers/examples/lexer.cpp similarity index 100% rename from src/Parsers/tests/lexer.cpp rename to src/Parsers/examples/lexer.cpp diff --git a/src/Parsers/tests/lexer_fuzzer.cpp b/src/Parsers/examples/lexer_fuzzer.cpp similarity index 100% rename from src/Parsers/tests/lexer_fuzzer.cpp rename to src/Parsers/examples/lexer_fuzzer.cpp diff --git a/src/Parsers/tests/select_parser.cpp b/src/Parsers/examples/select_parser.cpp similarity index 100% rename from src/Parsers/tests/select_parser.cpp rename to src/Parsers/examples/select_parser.cpp diff --git a/src/Parsers/tests/select_parser_fuzzer.cpp b/src/Parsers/examples/select_parser_fuzzer.cpp similarity index 100% rename from src/Parsers/tests/select_parser_fuzzer.cpp rename to src/Parsers/examples/select_parser_fuzzer.cpp diff --git a/src/Processors/CMakeLists.txt b/src/Processors/CMakeLists.txt index 99ba159eaf4..7e965188b4c 100644 --- a/src/Processors/CMakeLists.txt +++ b/src/Processors/CMakeLists.txt @@ -1,4 +1,4 @@ -if (ENABLE_TESTS) - add_subdirectory (tests) +if (ENABLE_EXAMPLES) + add_subdirectory(examples) endif () diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index f295fe00299..7152c9d9916 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -147,6 +147,13 @@ void ParallelParsingInputFormat::onBackgroundException(size_t offset) Chunk ParallelParsingInputFormat::generate() { + /// Delayed launching of segmentator thread + if (unlikely(!parsing_started.exchange(true))) + { + segmentator_thread = ThreadFromGlobalPool( + &ParallelParsingInputFormat::segmentatorThreadFunction, this, CurrentThread::getGroup()); + } + if (isCancelled() || parsing_finished) { /** diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 559507055b9..dafaf9bed72 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -97,9 +97,6 @@ public: // bump into reader thread on wraparound. processing_units.resize(params.max_threads + 2); - segmentator_thread = ThreadFromGlobalPool( - &ParallelParsingInputFormat::segmentatorThreadFunction, this, CurrentThread::getGroup()); - LOG_TRACE(&Poco::Logger::get("ParallelParsingInputFormat"), "Parallel parsing is used"); } @@ -205,6 +202,7 @@ private: Poco::Event first_parser_finished; + std::atomic parsing_started{false}; std::atomic parsing_finished{false}; /// There are multiple "parsers", that's why we use thread pool. diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index daa6e4981bb..772390acb32 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -163,6 +163,11 @@ void AggregatingStep::describeActions(FormatSettings & settings) const params.explain(settings.out, settings.offset); } +void AggregatingStep::describeActions(JSONBuilder::JSONMap & map) const +{ + params.explain(map); +} + void AggregatingStep::describePipeline(FormatSettings & settings) const { if (!aggregating.empty()) diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index c29a3cc2836..696aabd4de7 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -29,6 +29,8 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; + void describeActions(FormatSettings &) const override; void describePipeline(FormatSettings & settings) const override; diff --git a/src/Processors/QueryPlan/ArrayJoinStep.cpp b/src/Processors/QueryPlan/ArrayJoinStep.cpp index 7e82ec87353..9089bb8e5a2 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.cpp +++ b/src/Processors/QueryPlan/ArrayJoinStep.cpp @@ -5,7 +5,7 @@ #include #include #include - +#include namespace DB { @@ -87,4 +87,15 @@ void ArrayJoinStep::describeActions(FormatSettings & settings) const settings.out << '\n'; } +void ArrayJoinStep::describeActions(JSONBuilder::JSONMap & map) const +{ + map.add("Left", array_join->is_left); + + auto columns_array = std::make_unique(); + for (const auto & column : array_join->columns) + columns_array->add(column); + + map.add("Columns", std::move(columns_array)); +} + } diff --git a/src/Processors/QueryPlan/ArrayJoinStep.h b/src/Processors/QueryPlan/ArrayJoinStep.h index db1b4187549..b3e08c2023c 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.h +++ b/src/Processors/QueryPlan/ArrayJoinStep.h @@ -15,6 +15,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings & settings) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; void updateInputStream(DataStream input_stream, Block result_header); diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 9ea8e7b237b..9bcbc9d8b09 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -60,6 +61,15 @@ void CreatingSetStep::describeActions(FormatSettings & settings) const settings.out << description << '\n'; } +void CreatingSetStep::describeActions(JSONBuilder::JSONMap & map) const +{ + if (subquery_for_set.set) + map.add("Set", description); + else if (subquery_for_set.join) + map.add("Join", description); +} + + CreatingSetsStep::CreatingSetsStep(DataStreams input_streams_) { if (input_streams_.empty()) diff --git a/src/Processors/QueryPlan/CreatingSetsStep.h b/src/Processors/QueryPlan/CreatingSetsStep.h index c2b452ecdf5..fa6d34ef667 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.h +++ b/src/Processors/QueryPlan/CreatingSetsStep.h @@ -23,6 +23,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; private: diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp index ecd7918118b..5edd2f52f47 100644 --- a/src/Processors/QueryPlan/DistinctStep.cpp +++ b/src/Processors/QueryPlan/DistinctStep.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB { @@ -102,4 +103,13 @@ void DistinctStep::describeActions(FormatSettings & settings) const settings.out << '\n'; } +void DistinctStep::describeActions(JSONBuilder::JSONMap & map) const +{ + auto columns_array = std::make_unique(); + for (const auto & column : columns) + columns_array->add(column); + + map.add("Columns", std::move(columns_array)); +} + } diff --git a/src/Processors/QueryPlan/DistinctStep.h b/src/Processors/QueryPlan/DistinctStep.h index f4a5647fbc7..815601d6253 100644 --- a/src/Processors/QueryPlan/DistinctStep.h +++ b/src/Processors/QueryPlan/DistinctStep.h @@ -20,6 +20,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; private: diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp index c85092edf05..e6a0475a7c2 100644 --- a/src/Processors/QueryPlan/ExpressionStep.cpp +++ b/src/Processors/QueryPlan/ExpressionStep.cpp @@ -7,6 +7,8 @@ #include #include +#include + namespace DB { @@ -110,6 +112,12 @@ void ExpressionStep::describeActions(FormatSettings & settings) const settings.out << '\n'; } +void ExpressionStep::describeActions(JSONBuilder::JSONMap & map) const +{ + auto expression = std::make_shared(actions_dag, ExpressionActionsSettings{}); + map.add("Expression", expression->toTree()); +} + JoinStep::JoinStep(const DataStream & input_stream_, JoinPtr join_, bool has_non_joined_rows_, size_t max_block_size_) : ITransformingStep( input_stream_, diff --git a/src/Processors/QueryPlan/ExpressionStep.h b/src/Processors/QueryPlan/ExpressionStep.h index bcc1b0ef7b6..b7c4c0974f3 100644 --- a/src/Processors/QueryPlan/ExpressionStep.h +++ b/src/Processors/QueryPlan/ExpressionStep.h @@ -30,6 +30,8 @@ public: const ActionsDAGPtr & getExpression() const { return actions_dag; } + void describeActions(JSONBuilder::JSONMap & map) const override; + private: ActionsDAGPtr actions_dag; }; diff --git a/src/Processors/QueryPlan/FillingStep.cpp b/src/Processors/QueryPlan/FillingStep.cpp index 1bdd4fc468c..a4306ffed2b 100644 --- a/src/Processors/QueryPlan/FillingStep.cpp +++ b/src/Processors/QueryPlan/FillingStep.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB { @@ -50,4 +51,9 @@ void FillingStep::describeActions(FormatSettings & settings) const settings.out << '\n'; } +void FillingStep::describeActions(JSONBuilder::JSONMap & map) const +{ + map.add("Sort Description", explainSortDescription(sort_description, input_streams.front().header)); +} + } diff --git a/src/Processors/QueryPlan/FillingStep.h b/src/Processors/QueryPlan/FillingStep.h index 20eb132b7a6..f4c6782e9df 100644 --- a/src/Processors/QueryPlan/FillingStep.h +++ b/src/Processors/QueryPlan/FillingStep.h @@ -15,6 +15,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; const SortDescription & getSortDescription() const { return sort_description; } diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp index 6930d9563f9..522e7dabba8 100644 --- a/src/Processors/QueryPlan/FilterStep.cpp +++ b/src/Processors/QueryPlan/FilterStep.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -113,4 +114,13 @@ void FilterStep::describeActions(FormatSettings & settings) const settings.out << '\n'; } +void FilterStep::describeActions(JSONBuilder::JSONMap & map) const +{ + map.add("Filter Column", filter_column_name); + map.add("Removes Filter", remove_filter_column); + + auto expression = std::make_shared(actions_dag, ExpressionActionsSettings{}); + map.add("Expression", expression->toTree()); +} + } diff --git a/src/Processors/QueryPlan/FilterStep.h b/src/Processors/QueryPlan/FilterStep.h index 0cd48691261..d01d128a08c 100644 --- a/src/Processors/QueryPlan/FilterStep.h +++ b/src/Processors/QueryPlan/FilterStep.h @@ -22,6 +22,7 @@ public: void updateInputStream(DataStream input_stream, bool keep_header); + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; const ActionsDAGPtr & getExpression() const { return actions_dag; } diff --git a/src/Processors/QueryPlan/FinishSortingStep.cpp b/src/Processors/QueryPlan/FinishSortingStep.cpp index 489de74aa2d..a2e056b3029 100644 --- a/src/Processors/QueryPlan/FinishSortingStep.cpp +++ b/src/Processors/QueryPlan/FinishSortingStep.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -101,4 +102,13 @@ void FinishSortingStep::describeActions(FormatSettings & settings) const settings.out << prefix << "Limit " << limit << '\n'; } +void FinishSortingStep::describeActions(JSONBuilder::JSONMap & map) const +{ + map.add("Prefix Sort Description", explainSortDescription(prefix_description, input_streams.front().header)); + map.add("Result Sort Description", explainSortDescription(result_description, input_streams.front().header)); + + if (limit) + map.add("Limit", limit); +} + } diff --git a/src/Processors/QueryPlan/FinishSortingStep.h b/src/Processors/QueryPlan/FinishSortingStep.h index c89f8fee40d..9fe031e792d 100644 --- a/src/Processors/QueryPlan/FinishSortingStep.h +++ b/src/Processors/QueryPlan/FinishSortingStep.h @@ -20,6 +20,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; /// Add limit or change it to lower value. diff --git a/src/Processors/QueryPlan/IQueryPlanStep.h b/src/Processors/QueryPlan/IQueryPlanStep.h index 2974891e2bf..9ff2b22e5b8 100644 --- a/src/Processors/QueryPlan/IQueryPlanStep.h +++ b/src/Processors/QueryPlan/IQueryPlanStep.h @@ -3,6 +3,8 @@ #include #include +namespace JSONBuilder { class JSONMap; } + namespace DB { @@ -14,6 +16,8 @@ class IProcessor; using ProcessorPtr = std::shared_ptr; using Processors = std::vector; +namespace JSONBuilder { class JSONMap; } + /// Description of data stream. /// Single logical data stream may relate to many ports of pipeline. class DataStream @@ -97,9 +101,11 @@ public: }; /// Get detailed description of step actions. This is shown in EXPLAIN query with options `actions = 1`. + virtual void describeActions(JSONBuilder::JSONMap & /*map*/) const {} virtual void describeActions(FormatSettings & /*settings*/) const {} /// Get detailed description of read-from-storage step indexes (if any). Shown in with options `indexes = 1`. + virtual void describeIndexes(JSONBuilder::JSONMap & /*map*/) const {} virtual void describeIndexes(FormatSettings & /*settings*/) const {} /// Get description of processors added in current step. Should be called after updatePipeline(). diff --git a/src/Processors/QueryPlan/LimitByStep.cpp b/src/Processors/QueryPlan/LimitByStep.cpp index 78f96d75949..8ded0784b41 100644 --- a/src/Processors/QueryPlan/LimitByStep.cpp +++ b/src/Processors/QueryPlan/LimitByStep.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB { @@ -72,4 +73,15 @@ void LimitByStep::describeActions(FormatSettings & settings) const settings.out << prefix << "Offset " << group_offset << '\n'; } +void LimitByStep::describeActions(JSONBuilder::JSONMap & map) const +{ + auto columns_array = std::make_unique(); + for (const auto & column : columns) + columns_array->add(column); + + map.add("Columns", std::move(columns_array)); + map.add("Length", group_length); + map.add("Offset", group_offset); +} + } diff --git a/src/Processors/QueryPlan/LimitByStep.h b/src/Processors/QueryPlan/LimitByStep.h index 7b978286e48..1b574cd02a1 100644 --- a/src/Processors/QueryPlan/LimitByStep.h +++ b/src/Processors/QueryPlan/LimitByStep.h @@ -16,6 +16,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; private: diff --git a/src/Processors/QueryPlan/LimitStep.cpp b/src/Processors/QueryPlan/LimitStep.cpp index 4c6b4523a54..5f5a0bd0d64 100644 --- a/src/Processors/QueryPlan/LimitStep.cpp +++ b/src/Processors/QueryPlan/LimitStep.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB { @@ -76,4 +77,12 @@ void LimitStep::describeActions(FormatSettings & settings) const } } +void LimitStep::describeActions(JSONBuilder::JSONMap & map) const +{ + map.add("Limit", limit); + map.add("Offset", offset); + map.add("With Ties", with_ties); + map.add("Reads All Data", always_read_till_end); +} + } diff --git a/src/Processors/QueryPlan/LimitStep.h b/src/Processors/QueryPlan/LimitStep.h index 4428acb9ef0..772ba0722a7 100644 --- a/src/Processors/QueryPlan/LimitStep.h +++ b/src/Processors/QueryPlan/LimitStep.h @@ -20,6 +20,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; size_t getLimitForSorting() const diff --git a/src/Processors/QueryPlan/MergeSortingStep.cpp b/src/Processors/QueryPlan/MergeSortingStep.cpp index 8df363ffa87..c9e141281f4 100644 --- a/src/Processors/QueryPlan/MergeSortingStep.cpp +++ b/src/Processors/QueryPlan/MergeSortingStep.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB { @@ -84,4 +85,12 @@ void MergeSortingStep::describeActions(FormatSettings & settings) const settings.out << prefix << "Limit " << limit << '\n'; } +void MergeSortingStep::describeActions(JSONBuilder::JSONMap & map) const +{ + map.add("Sort Description", explainSortDescription(description, input_streams.front().header)); + + if (limit) + map.add("Limit", limit); +} + } diff --git a/src/Processors/QueryPlan/MergeSortingStep.h b/src/Processors/QueryPlan/MergeSortingStep.h index 9802d1a760e..dcecdffd122 100644 --- a/src/Processors/QueryPlan/MergeSortingStep.h +++ b/src/Processors/QueryPlan/MergeSortingStep.h @@ -26,6 +26,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; /// Add limit or change it to lower value. diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.cpp b/src/Processors/QueryPlan/MergingAggregatedStep.cpp index 56e91dcbd71..71efb37b363 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.cpp +++ b/src/Processors/QueryPlan/MergingAggregatedStep.cpp @@ -68,4 +68,9 @@ void MergingAggregatedStep::describeActions(FormatSettings & settings) const return params->params.explain(settings.out, settings.offset); } +void MergingAggregatedStep::describeActions(JSONBuilder::JSONMap & map) const +{ + params->params.explain(map); +} + } diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.h b/src/Processors/QueryPlan/MergingAggregatedStep.h index 2742a040708..2e94d536a8c 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.h +++ b/src/Processors/QueryPlan/MergingAggregatedStep.h @@ -23,6 +23,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; private: diff --git a/src/Processors/QueryPlan/MergingFinal.cpp b/src/Processors/QueryPlan/MergingFinal.cpp index 4109c5e7274..c564a28d377 100644 --- a/src/Processors/QueryPlan/MergingFinal.cpp +++ b/src/Processors/QueryPlan/MergingFinal.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB { @@ -161,4 +162,9 @@ void MergingFinal::describeActions(FormatSettings & settings) const settings.out << '\n'; } +void MergingFinal::describeActions(JSONBuilder::JSONMap & map) const +{ + map.add("Sort Description", explainSortDescription(sort_description, input_streams.front().header)); +} + } diff --git a/src/Processors/QueryPlan/MergingFinal.h b/src/Processors/QueryPlan/MergingFinal.h index 5b621e9f4b2..ed0394a62f4 100644 --- a/src/Processors/QueryPlan/MergingFinal.h +++ b/src/Processors/QueryPlan/MergingFinal.h @@ -22,6 +22,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; private: diff --git a/src/Processors/QueryPlan/MergingSortedStep.cpp b/src/Processors/QueryPlan/MergingSortedStep.cpp index e31644ac655..7e866f4ccd2 100644 --- a/src/Processors/QueryPlan/MergingSortedStep.cpp +++ b/src/Processors/QueryPlan/MergingSortedStep.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB { @@ -73,4 +74,12 @@ void MergingSortedStep::describeActions(FormatSettings & settings) const settings.out << prefix << "Limit " << limit << '\n'; } +void MergingSortedStep::describeActions(JSONBuilder::JSONMap & map) const +{ + map.add("Sort Description", explainSortDescription(sort_description, input_streams.front().header)); + + if (limit) + map.add("Limit", limit); +} + } diff --git a/src/Processors/QueryPlan/MergingSortedStep.h b/src/Processors/QueryPlan/MergingSortedStep.h index ac5a7f666c8..4f82e3830d0 100644 --- a/src/Processors/QueryPlan/MergingSortedStep.h +++ b/src/Processors/QueryPlan/MergingSortedStep.h @@ -21,6 +21,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; /// Add limit or change it to lower value. diff --git a/src/Processors/QueryPlan/OffsetStep.cpp b/src/Processors/QueryPlan/OffsetStep.cpp index b455a32d5af..34ddb687ddd 100644 --- a/src/Processors/QueryPlan/OffsetStep.cpp +++ b/src/Processors/QueryPlan/OffsetStep.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB { @@ -41,4 +42,9 @@ void OffsetStep::describeActions(FormatSettings & settings) const settings.out << String(settings.offset, ' ') << "Offset " << offset << '\n'; } +void OffsetStep::describeActions(JSONBuilder::JSONMap & map) const +{ + map.add("Offset", offset); +} + } diff --git a/src/Processors/QueryPlan/OffsetStep.h b/src/Processors/QueryPlan/OffsetStep.h index c278744ff50..a10fcc7baec 100644 --- a/src/Processors/QueryPlan/OffsetStep.h +++ b/src/Processors/QueryPlan/OffsetStep.h @@ -15,6 +15,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; private: diff --git a/src/Processors/QueryPlan/PartialSortingStep.cpp b/src/Processors/QueryPlan/PartialSortingStep.cpp index 5a4d041a3d1..f4abea440fe 100644 --- a/src/Processors/QueryPlan/PartialSortingStep.cpp +++ b/src/Processors/QueryPlan/PartialSortingStep.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -81,4 +82,12 @@ void PartialSortingStep::describeActions(FormatSettings & settings) const settings.out << prefix << "Limit " << limit << '\n'; } +void PartialSortingStep::describeActions(JSONBuilder::JSONMap & map) const +{ + map.add("Sort Description", explainSortDescription(sort_description, input_streams.front().header)); + + if (limit) + map.add("Limit", limit); +} + } diff --git a/src/Processors/QueryPlan/PartialSortingStep.h b/src/Processors/QueryPlan/PartialSortingStep.h index e93e9e8ef93..aeca42f7096 100644 --- a/src/Processors/QueryPlan/PartialSortingStep.h +++ b/src/Processors/QueryPlan/PartialSortingStep.h @@ -20,6 +20,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; /// Add limit or change it to lower value. diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index ad3649385fd..3e46adb9d9c 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB { @@ -200,6 +201,92 @@ void QueryPlan::addInterpreterContext(std::shared_ptr context) } +static void explainStep(const IQueryPlanStep & step, JSONBuilder::JSONMap & map, const QueryPlan::ExplainPlanOptions & options) +{ + map.add("Node Type", step.getName()); + + if (options.description) + { + const auto & description = step.getStepDescription(); + if (!description.empty()) + map.add("Description", description); + } + + if (options.header && step.hasOutputStream()) + { + auto header_array = std::make_unique(); + + for (const auto & output_column : step.getOutputStream().header) + { + auto column_map = std::make_unique(); + column_map->add("Name", output_column.name); + if (output_column.type) + column_map->add("Type", output_column.type->getName()); + + header_array->add(std::move(column_map)); + } + + map.add("Header", std::move(header_array)); + } + + if (options.actions) + step.describeActions(map); + + if (options.indexes) + step.describeIndexes(map); +} + +JSONBuilder::ItemPtr QueryPlan::explainPlan(const ExplainPlanOptions & options) +{ + checkInitialized(); + + struct Frame + { + Node * node; + size_t next_child = 0; + std::unique_ptr node_map = {}; + std::unique_ptr children_array = {}; + }; + + std::stack stack; + stack.push(Frame{.node = root}); + + std::unique_ptr tree; + + while (!stack.empty()) + { + auto & frame = stack.top(); + + if (frame.next_child == 0) + { + if (!frame.node->children.empty()) + frame.children_array = std::make_unique(); + + frame.node_map = std::make_unique(); + explainStep(*frame.node->step, *frame.node_map, options); + } + + if (frame.next_child < frame.node->children.size()) + { + stack.push(Frame{frame.node->children[frame.next_child]}); + ++frame.next_child; + } + else + { + if (frame.children_array) + frame.node_map->add("Plans", std::move(frame.children_array)); + + tree.swap(frame.node_map); + stack.pop(); + + if (!stack.empty()) + stack.top().children_array->add(std::move(tree)); + } + } + + return tree; +} + static void explainStep( const IQueryPlanStep & step, IQueryPlanStep::FormatSettings & settings, diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index 901d83c3ab8..4c75f00cf4d 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -29,6 +29,12 @@ class Pipe; struct QueryPlanOptimizationSettings; struct BuildQueryPipelineSettings; +namespace JSONBuilder +{ + class IItem; + using ItemPtr = std::unique_ptr; +} + /// A tree of query steps. /// The goal of QueryPlan is to build QueryPipeline. /// QueryPlan let delay pipeline creation which is helpful for pipeline-level optimizations. @@ -76,6 +82,7 @@ public: bool header = false; }; + JSONBuilder::ItemPtr explainPlan(const ExplainPlanOptions & options); void explainPlan(WriteBuffer & buffer, const ExplainPlanOptions & options); void explainPipeline(WriteBuffer & buffer, const ExplainPipelineOptions & options); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index ebf9c9e4121..8a7fca63d8c 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -198,16 +199,27 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const } } +void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const +{ + map.add("Read Type", readTypeToString(read_type)); + if (index_stats && !index_stats->empty()) + { + map.add("Parts", index_stats->back().num_parts_after); + map.add("Granules", index_stats->back().num_granules_after); + } +} + void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const { std::string prefix(format_settings.offset, format_settings.indent_char); if (index_stats && !index_stats->empty()) { - std::string indent(format_settings.indent, format_settings.indent_char); - /// Do not print anything if no indexes is applied. - if (index_stats->size() > 1 || index_stats->front().type != IndexType::None) - format_settings.out << prefix << "Indexes:\n"; + if (index_stats->size() == 1 && index_stats->front().type == IndexType::None) + return; + + std::string indent(format_settings.indent, format_settings.indent_char); + format_settings.out << prefix << "Indexes:\n"; for (size_t i = 0; i < index_stats->size(); ++i) { @@ -246,4 +258,58 @@ void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const } } +void ReadFromMergeTree::describeIndexes(JSONBuilder::JSONMap & map) const +{ + if (index_stats && !index_stats->empty()) + { + /// Do not print anything if no indexes is applied. + if (index_stats->size() == 1 && index_stats->front().type == IndexType::None) + return; + + auto indexes_array = std::make_unique(); + + for (size_t i = 0; i < index_stats->size(); ++i) + { + const auto & stat = (*index_stats)[i]; + if (stat.type == IndexType::None) + continue; + + auto index_map = std::make_unique(); + + index_map->add("Type", indexTypeToString(stat.type)); + + if (!stat.name.empty()) + index_map->add("Name", stat.name); + + if (!stat.description.empty()) + index_map->add("Description", stat.description); + + if (!stat.used_keys.empty()) + { + auto keys_array = std::make_unique(); + + for (const auto & used_key : stat.used_keys) + keys_array->add(used_key); + + index_map->add("Keys", std::move(keys_array)); + } + + if (!stat.condition.empty()) + index_map->add("Condition", stat.condition); + + if (i) + index_map->add("Initial Parts", (*index_stats)[i - 1].num_parts_after); + index_map->add("Selected Parts", stat.num_parts_after); + + if (i) + index_map->add("Initial Granules", (*index_stats)[i - 1].num_granules_after); + index_map->add("Selected Granules", stat.num_granules_after); + + indexes_array->add(std::move(index_map)); + } + + map.add("Indexes", std::move(indexes_array)); + } +} + } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 1d6a4491588..479610b3edc 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -87,6 +87,9 @@ public: void describeActions(FormatSettings & format_settings) const override; void describeIndexes(FormatSettings & format_settings) const override; + void describeActions(JSONBuilder::JSONMap & map) const override; + void describeIndexes(JSONBuilder::JSONMap & map) const override; + private: const MergeTreeData & storage; StorageMetadataPtr metadata_snapshot; diff --git a/src/Processors/QueryPlan/TotalsHavingStep.cpp b/src/Processors/QueryPlan/TotalsHavingStep.cpp index 7a60f0a6f36..4966c04dee7 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.cpp +++ b/src/Processors/QueryPlan/TotalsHavingStep.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -95,4 +96,15 @@ void TotalsHavingStep::describeActions(FormatSettings & settings) const } } +void TotalsHavingStep::describeActions(JSONBuilder::JSONMap & map) const +{ + map.add("Mode", totalsModeToString(totals_mode, auto_include_threshold)); + if (actions_dag) + { + map.add("Filter column", filter_column_name); + auto expression = std::make_shared(actions_dag, ExpressionActionsSettings{}); + map.add("Expression", expression->toTree()); + } +} + } diff --git a/src/Processors/QueryPlan/TotalsHavingStep.h b/src/Processors/QueryPlan/TotalsHavingStep.h index 4041960de28..bc053c96970 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.h +++ b/src/Processors/QueryPlan/TotalsHavingStep.h @@ -26,6 +26,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings & settings) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; const ActionsDAGPtr & getActions() const { return actions_dag; } diff --git a/src/Processors/QueryPlan/WindowStep.cpp b/src/Processors/QueryPlan/WindowStep.cpp index 66c329acb4b..29f2999ec83 100644 --- a/src/Processors/QueryPlan/WindowStep.cpp +++ b/src/Processors/QueryPlan/WindowStep.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -116,4 +117,25 @@ void WindowStep::describeActions(FormatSettings & settings) const } } +void WindowStep::describeActions(JSONBuilder::JSONMap & map) const +{ + if (!window_description.partition_by.empty()) + { + auto partion_columns_array = std::make_unique(); + for (const auto & descr : window_description.partition_by) + partion_columns_array->add(descr.column_name); + + map.add("Partition By", std::move(partion_columns_array)); + } + + if (!window_description.order_by.empty()) + map.add("Sort Description", explainSortDescription(window_description.order_by, {})); + + auto functions_array = std::make_unique(); + for (const auto & func : window_functions) + functions_array->add(func.column_name); + + map.add("Functions", std::move(functions_array)); +} + } diff --git a/src/Processors/QueryPlan/WindowStep.h b/src/Processors/QueryPlan/WindowStep.h index 9e3c18c0b16..b5018b1d5a7 100644 --- a/src/Processors/QueryPlan/WindowStep.h +++ b/src/Processors/QueryPlan/WindowStep.h @@ -22,6 +22,7 @@ public: void transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) override; + void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; private: diff --git a/src/Functions/tests/CMakeLists.txt b/src/Processors/examples/CMakeLists.txt similarity index 100% rename from src/Functions/tests/CMakeLists.txt rename to src/Processors/examples/CMakeLists.txt diff --git a/src/Processors/tests/processors_test_aggregation.cpp b/src/Processors/examples/processors_test_aggregation.cpp similarity index 100% rename from src/Processors/tests/processors_test_aggregation.cpp rename to src/Processors/examples/processors_test_aggregation.cpp diff --git a/src/Processors/tests/processors_test_merge_sorting_transform.cpp b/src/Processors/examples/processors_test_merge_sorting_transform.cpp similarity index 100% rename from src/Processors/tests/processors_test_merge_sorting_transform.cpp rename to src/Processors/examples/processors_test_merge_sorting_transform.cpp diff --git a/src/Processors/tests/CMakeLists.txt b/src/Processors/tests/CMakeLists.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/Storages/CMakeLists.txt b/src/Storages/CMakeLists.txt index deb1c9f6716..ff22e9fa9e1 100644 --- a/src/Storages/CMakeLists.txt +++ b/src/Storages/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(MergeTree) add_subdirectory(System) -if(ENABLE_TESTS) - add_subdirectory(tests) +if(ENABLE_EXAMPLES) + add_subdirectory(examples) endif() diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index e48e9e49919..a0fb7c70843 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -108,6 +108,9 @@ public: /// Returns true if the storage is a view of a table or another view. virtual bool isView() const { return false; } + /// Returns true if the storage is dictionary + virtual bool isDictionary() const { return false; } + /// Returns true if the storage supports queries with the SAMPLE section. virtual bool supportsSampling() const { return getInMemoryMetadataPtr()->hasSamplingKey(); } diff --git a/src/Storages/MergeTree/CMakeLists.txt b/src/Storages/MergeTree/CMakeLists.txt index 36cab0b3590..390835f17ae 100644 --- a/src/Storages/MergeTree/CMakeLists.txt +++ b/src/Storages/MergeTree/CMakeLists.txt @@ -1,3 +1,3 @@ -if(ENABLE_TESTS) - add_subdirectory(tests) +if(ENABLE_EXAMPLES) + add_subdirectory(examples) endif() diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index a7826bc6ce9..f0d23041279 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -24,6 +24,8 @@ #include #include +#include + namespace DB { @@ -91,7 +93,7 @@ static String extractFixedPrefixFromLikePattern(const String & like_pattern) */ static String firstStringThatIsGreaterThanAllStringsWithPrefix(const String & prefix) { - /** Increment the last byte of the prefix by one. But if it is 255, then remove it and increase the previous one. + /** Increment the last byte of the prefix by one. But if it is max (255), then remove it and increase the previous one. * Example (for convenience, suppose that the maximum value of byte is `z`) * abcx -> abcy * abcz -> abd @@ -101,7 +103,7 @@ static String firstStringThatIsGreaterThanAllStringsWithPrefix(const String & pr String res = prefix; - while (!res.empty() && static_cast(res.back()) == 255) + while (!res.empty() && static_cast(res.back()) == std::numeric_limits::max()) res.pop_back(); if (res.empty()) @@ -1346,7 +1348,7 @@ KeyCondition::Description KeyCondition::getDescription() const Or, }; - Type type; + Type type{}; /// Only for Leaf const RPNElement * element = nullptr; diff --git a/src/Storages/MergeTree/ReplicatedFetchList.cpp b/src/Storages/MergeTree/ReplicatedFetchList.cpp index 82bc8ae21e0..6d4451ad3e0 100644 --- a/src/Storages/MergeTree/ReplicatedFetchList.cpp +++ b/src/Storages/MergeTree/ReplicatedFetchList.cpp @@ -39,7 +39,6 @@ ReplicatedFetchInfo ReplicatedFetchListElement::getInfo() const res.source_replica_port = source_replica_port; res.interserver_scheme = interserver_scheme; res.uri = uri; - res.interserver_scheme = interserver_scheme; res.to_detached = to_detached; res.elapsed = watch.elapsedSeconds(); res.progress = progress.load(std::memory_order_relaxed); diff --git a/src/Storages/MergeTree/tests/CMakeLists.txt b/src/Storages/MergeTree/examples/CMakeLists.txt similarity index 100% rename from src/Storages/MergeTree/tests/CMakeLists.txt rename to src/Storages/MergeTree/examples/CMakeLists.txt diff --git a/src/Storages/MergeTree/tests/wal_action_metadata.cpp b/src/Storages/MergeTree/examples/wal_action_metadata.cpp similarity index 100% rename from src/Storages/MergeTree/tests/wal_action_metadata.cpp rename to src/Storages/MergeTree/examples/wal_action_metadata.cpp diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 862747abcb9..0cff7e00bc6 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -762,8 +762,8 @@ static StoragePtr create(const StorageFactory::Arguments & args) { for (size_t i = 0; i < data_types.size(); ++i) if (isFloat(data_types[i])) - throw Exception( - "Donot support float point as partition key: " + metadata.partition_key.column_names[i], ErrorCodes::BAD_ARGUMENTS); + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Floating point partition key is not supported: {}", metadata.partition_key.column_names[i]); } if (arg_num != arg_cnt) diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index e2cab153092..16818c9ea18 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -5,11 +5,13 @@ #include #include #include +#include #include #include #include #include #include +#include namespace DB @@ -19,6 +21,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int THERE_IS_NO_COLUMN; extern const int CANNOT_DETACH_DICTIONARY_AS_TABLE; + extern const int DICTIONARY_ALREADY_EXISTS; } namespace @@ -93,8 +96,10 @@ StorageDictionary::StorageDictionary( const StorageID & table_id_, const String & dictionary_name_, const ColumnsDescription & columns_, - Location location_) + Location location_, + ContextPtr context_) : IStorage(table_id_) + , WithContext(context_->getGlobalContext()) , dictionary_name(dictionary_name_) , location(location_) { @@ -105,18 +110,52 @@ StorageDictionary::StorageDictionary( StorageDictionary::StorageDictionary( - const StorageID & table_id_, const String & dictionary_name_, const DictionaryStructure & dictionary_structure_, Location location_) - : StorageDictionary(table_id_, dictionary_name_, ColumnsDescription{getNamesAndTypes(dictionary_structure_)}, location_) + const StorageID & table_id_, + const String & dictionary_name_, + const DictionaryStructure & dictionary_structure_, + Location location_, + ContextPtr context_) + : StorageDictionary( + table_id_, + dictionary_name_, + ColumnsDescription{getNamesAndTypes(dictionary_structure_)}, + location_, + context_) { } +StorageDictionary::StorageDictionary( + const StorageID & table_id, + LoadablesConfigurationPtr dictionary_configuration, + ContextPtr context_) + : StorageDictionary( + table_id, + table_id.getFullNameNotQuoted(), + context_->getExternalDictionariesLoader().getDictionaryStructure(*dictionary_configuration), + Location::SameDatabaseAndNameAsDictionary, + context_) +{ + configuration = dictionary_configuration; + + auto repository = std::make_unique(*this); + remove_repository_callback = context_->getExternalDictionariesLoader().addConfigRepository(std::move(repository)); +} + +StorageDictionary::~StorageDictionary() +{ + removeDictionaryConfigurationFromRepository(); +} void StorageDictionary::checkTableCanBeDropped() const { if (location == Location::SameDatabaseAndNameAsDictionary) - throw Exception("Cannot drop/detach dictionary " + backQuote(dictionary_name) + " as table, use DROP DICTIONARY or DETACH DICTIONARY query instead", ErrorCodes::CANNOT_DETACH_DICTIONARY_AS_TABLE); + throw Exception(ErrorCodes::CANNOT_DETACH_DICTIONARY_AS_TABLE, + "Cannot drop/detach dictionary {} as table, use DROP DICTIONARY or DETACH DICTIONARY query instead", + dictionary_name); if (location == Location::DictionaryDatabase) - throw Exception("Cannot drop/detach table " + getStorageID().getFullTableName() + " from a database with DICTIONARY engine", ErrorCodes::CANNOT_DETACH_DICTIONARY_AS_TABLE); + throw Exception(ErrorCodes::CANNOT_DETACH_DICTIONARY_AS_TABLE, + "Cannot drop/detach table from a database with DICTIONARY engine, use DROP DICTIONARY or DETACH DICTIONARY query instead", + dictionary_name); } void StorageDictionary::checkTableCanBeDetached() const @@ -128,37 +167,130 @@ Pipe StorageDictionary::read( const Names & column_names, const StorageMetadataPtr & /*metadata_snapshot*/, SelectQueryInfo & /*query_info*/, - ContextPtr context, + ContextPtr local_context, QueryProcessingStage::Enum /*processed_stage*/, const size_t max_block_size, const unsigned /*threads*/) { - auto dictionary = context->getExternalDictionariesLoader().getDictionary(dictionary_name, context); + auto dictionary = getContext()->getExternalDictionariesLoader().getDictionary(dictionary_name, local_context); auto stream = dictionary->getBlockInputStream(column_names, max_block_size); /// TODO: update dictionary interface for processors. return Pipe(std::make_shared(stream)); } +void StorageDictionary::shutdown() +{ + removeDictionaryConfigurationFromRepository(); +} + +void StorageDictionary::startup() +{ + auto global_context = getContext(); + + bool lazy_load = global_context->getConfigRef().getBool("dictionaries_lazy_load", true); + if (!lazy_load) + { + auto & external_dictionaries_loader = global_context->getExternalDictionariesLoader(); + + /// reloadConfig() is called here to force loading the dictionary. + external_dictionaries_loader.reloadConfig(getStorageID().getInternalDictionaryName()); + } +} + +void StorageDictionary::removeDictionaryConfigurationFromRepository() +{ + if (remove_repository_callback_executed) + return; + + remove_repository_callback_executed = true; + remove_repository_callback.reset(); +} + +Poco::Timestamp StorageDictionary::getUpdateTime() const +{ + std::lock_guard lock(dictionary_config_mutex); + return update_time; +} + +LoadablesConfigurationPtr StorageDictionary::getConfiguration() const +{ + std::lock_guard lock(dictionary_config_mutex); + return configuration; +} + +void StorageDictionary::renameInMemory(const StorageID & new_table_id) +{ + if (configuration) + { + configuration->setString("dictionary.database", new_table_id.database_name); + configuration->setString("dictionary.name", new_table_id.table_name); + + auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader(); + external_dictionaries_loader.reloadConfig(getStorageID().getInternalDictionaryName()); + + auto result = external_dictionaries_loader.getLoadResult(getStorageID().getInternalDictionaryName()); + if (!result.object) + return; + + const auto dictionary = std::static_pointer_cast(result.object); + dictionary->updateDictionaryName(new_table_id); + } + + IStorage::renameInMemory(new_table_id); +} void registerStorageDictionary(StorageFactory & factory) { factory.registerStorage("Dictionary", [](const StorageFactory::Arguments & args) { - if (args.engine_args.size() != 1) - throw Exception("Storage Dictionary requires single parameter: name of dictionary", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + auto query = args.query; - args.engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(args.engine_args[0], args.getLocalContext()); - String dictionary_name = args.engine_args[0]->as().value.safeGet(); + auto local_context = args.getLocalContext(); - if (!args.attach) + if (query.is_dictionary) { - const auto & dictionary = args.getContext()->getExternalDictionariesLoader().getDictionary(dictionary_name, args.getContext()); - const DictionaryStructure & dictionary_structure = dictionary->getStructure(); - checkNamesAndTypesCompatibleWithDictionary(dictionary_name, args.columns, dictionary_structure); - } + auto dictionary_id = args.table_id; + auto & external_dictionaries_loader = local_context->getExternalDictionariesLoader(); - return StorageDictionary::create(args.table_id, dictionary_name, args.columns, StorageDictionary::Location::Custom); + /// A dictionary with the same full name could be defined in *.xml config files. + if (external_dictionaries_loader.getCurrentStatus(dictionary_id.getFullNameNotQuoted()) != ExternalLoader::Status::NOT_EXIST) + throw Exception(ErrorCodes::DICTIONARY_ALREADY_EXISTS, + "Dictionary {} already exists.", dictionary_id.getFullNameNotQuoted()); + + /// Create dictionary storage that owns underlying dictionary + auto abstract_dictionary_configuration = getDictionaryConfigurationFromAST(args.query, local_context, dictionary_id.database_name); + auto result_storage = StorageDictionary::create(dictionary_id, abstract_dictionary_configuration, local_context); + + bool lazy_load = local_context->getConfigRef().getBool("dictionaries_lazy_load", true); + if (!args.attach && !lazy_load) + { + /// load() is called here to force loading the dictionary, wait until the loading is finished, + /// and throw an exception if the loading is failed. + external_dictionaries_loader.load(dictionary_id.getInternalDictionaryName()); + } + + return result_storage; + } + else + { + /// Create dictionary storage that is view of underlying dictionary + + if (args.engine_args.size() != 1) + throw Exception("Storage Dictionary requires single parameter: name of dictionary", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + args.engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(args.engine_args[0], local_context); + String dictionary_name = args.engine_args[0]->as().value.safeGet(); + + if (!args.attach) + { + const auto & dictionary = args.getContext()->getExternalDictionariesLoader().getDictionary(dictionary_name, args.getContext()); + const DictionaryStructure & dictionary_structure = dictionary->getStructure(); + checkNamesAndTypesCompatibleWithDictionary(dictionary_name, args.columns, dictionary_structure); + } + + return StorageDictionary::create(args.table_id, dictionary_name, args.columns, StorageDictionary::Location::Custom, local_context); + } }); } diff --git a/src/Storages/StorageDictionary.h b/src/Storages/StorageDictionary.h index 9e8564d4349..c22c337d40a 100644 --- a/src/Storages/StorageDictionary.h +++ b/src/Storages/StorageDictionary.h @@ -1,21 +1,26 @@ #pragma once -#include +#include #include +#include +#include + namespace DB { struct DictionaryStructure; class TableFunctionDictionary; -class StorageDictionary final : public ext::shared_ptr_helper, public IStorage +class StorageDictionary final : public ext::shared_ptr_helper, public IStorage, public WithContext { friend struct ext::shared_ptr_helper; friend class TableFunctionDictionary; public: std::string getName() const override { return "Dictionary"; } + ~StorageDictionary() override; + void checkTableCanBeDropped() const override; void checkTableCanBeDetached() const override; @@ -31,7 +36,16 @@ public: static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure); static String generateNamesAndTypesDescription(const NamesAndTypesList & list); - const String & dictionaryName() const { return dictionary_name; } + bool isDictionary() const override { return true; } + void shutdown() override; + void startup() override; + + void renameInMemory(const StorageID & new_table_id) override; + + Poco::Timestamp getUpdateTime() const; + LoadablesConfigurationPtr getConfiguration() const; + + const String & getDictionaryName() const { return dictionary_name; } /// Specifies where the table is located relative to the dictionary. enum class Location @@ -55,18 +69,33 @@ private: const String dictionary_name; const Location location; -protected: + mutable std::mutex dictionary_config_mutex; + Poco::Timestamp update_time; + LoadablesConfigurationPtr configuration; + + std::atomic remove_repository_callback_executed = false; + ext::scope_guard remove_repository_callback; + + void removeDictionaryConfigurationFromRepository(); + StorageDictionary( const StorageID & table_id_, const String & dictionary_name_, const ColumnsDescription & columns_, - Location location_); + Location location_, + ContextPtr context_); StorageDictionary( const StorageID & table_id_, const String & dictionary_name_, const DictionaryStructure & dictionary_structure, - Location location_); + Location location_, + ContextPtr context_); + + StorageDictionary( + const StorageID & table_id_, + LoadablesConfigurationPtr dictionary_configuration_, + ContextPtr context_); }; } diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp index 3489de0161a..7c1bcd1e83a 100644 --- a/src/Storages/StorageExternalDistributed.cpp +++ b/src/Storages/StorageExternalDistributed.cpp @@ -1,6 +1,5 @@ #include "StorageExternalDistributed.h" -#if USE_MYSQL || USE_LIBPQXX #include #include @@ -14,6 +13,7 @@ #include #include #include +#include #include @@ -48,6 +48,8 @@ StorageExternalDistributed::StorageExternalDistributed( std::vector shards_descriptions = parseRemoteDescription(cluster_description, 0, cluster_description.size(), ',', max_addresses); std::vector> addresses; +#if USE_MYSQL || USE_LIBPQXX + /// For each shard pass replicas description into storage, replicas are managed by storage's PoolWithFailover. for (const auto & shard_description : shards_descriptions) { @@ -100,8 +102,76 @@ StorageExternalDistributed::StorageExternalDistributed( } #endif default: + { throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Unsupported table engine. Supported engines are: MySQL, PostgreSQL"); + "Unsupported table engine. Supported engines are: MySQL, PostgreSQL, URL"); + } + } + + shards.emplace(std::move(shard)); + } + +#else + (void)table_engine; + (void)remote_database; + (void)remote_table; + (void)username; + (void)password; + (void)shards_descriptions; + (void)addresses; +#endif +} + + +StorageExternalDistributed::StorageExternalDistributed( + const String & addresses_description, + const StorageID & table_id, + const String & format_name, + const std::optional & format_settings, + const String & compression_method, + const ColumnsDescription & columns, + const ConstraintsDescription & constraints, + ContextPtr context) + : IStorage(table_id) +{ + StorageInMemoryMetadata storage_metadata; + storage_metadata.setColumns(columns); + storage_metadata.setConstraints(constraints); + setInMemoryMetadata(storage_metadata); + + size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; + /// Generate addresses without splitting for failover options + std::vector url_descriptions = parseRemoteDescription(addresses_description, 0, addresses_description.size(), ',', max_addresses); + std::vector uri_options; + + for (const auto & url_description : url_descriptions) + { + /// For each uri (which acts like shard) check if it has failover options + uri_options = parseRemoteDescription(url_description, 0, url_description.size(), '|', max_addresses); + StoragePtr shard; + + if (uri_options.size() > 1) + { + shard = std::make_shared( + uri_options, + table_id, + format_name, + format_settings, + columns, constraints, context, + compression_method); + } + else + { + Poco::URI uri(url_description); + shard = std::make_shared( + uri, + table_id, + format_name, + format_settings, + columns, constraints, context, + compression_method); + + LOG_DEBUG(&Poco::Logger::get("StorageURLDistributed"), "Adding URL: {}", url_description); } shards.emplace(std::move(shard)); @@ -151,39 +221,62 @@ void registerStorageExternalDistributed(StorageFactory & factory) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, args.getLocalContext()); const String & engine_name = engine_args[0]->as().value.safeGet(); - const String & cluster_description = engine_args[1]->as().value.safeGet(); - const String & remote_database = engine_args[2]->as().value.safeGet(); - const String & remote_table = engine_args[3]->as().value.safeGet(); - const String & username = engine_args[4]->as().value.safeGet(); - const String & password = engine_args[5]->as().value.safeGet(); + const String & addresses_description = engine_args[1]->as().value.safeGet(); StorageExternalDistributed::ExternalStorageEngine table_engine; - if (engine_name == "MySQL") - table_engine = StorageExternalDistributed::ExternalStorageEngine::MySQL; - else if (engine_name == "PostgreSQL") - table_engine = StorageExternalDistributed::ExternalStorageEngine::PostgreSQL; - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "External storage engine {} is not supported for StorageExternalDistributed. Supported engines are: MySQL, PostgreSQL", - engine_name); + if (engine_name == "URL") + { + table_engine = StorageExternalDistributed::ExternalStorageEngine::URL; - return StorageExternalDistributed::create( - args.table_id, - table_engine, - cluster_description, - remote_database, - remote_table, - username, - password, - args.columns, - args.constraints, - args.getContext()); + const String & format_name = engine_args[2]->as().value.safeGet(); + String compression_method = "auto"; + if (engine_args.size() == 4) + compression_method = engine_args[3]->as().value.safeGet(); + + auto format_settings = StorageURL::getFormatSettingsFromArgs(args); + + return StorageExternalDistributed::create( + addresses_description, + args.table_id, + format_name, + format_settings, + compression_method, + args.columns, + args.constraints, + args.getContext()); + } + else + { + if (engine_name == "MySQL") + table_engine = StorageExternalDistributed::ExternalStorageEngine::MySQL; + else if (engine_name == "PostgreSQL") + table_engine = StorageExternalDistributed::ExternalStorageEngine::PostgreSQL; + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "External storage engine {} is not supported for StorageExternalDistributed. Supported engines are: MySQL, PostgreSQL, URL", + engine_name); + + const String & remote_database = engine_args[2]->as().value.safeGet(); + const String & remote_table = engine_args[3]->as().value.safeGet(); + const String & username = engine_args[4]->as().value.safeGet(); + const String & password = engine_args[5]->as().value.safeGet(); + + return StorageExternalDistributed::create( + args.table_id, + table_engine, + addresses_description, + remote_database, + remote_table, + username, + password, + args.columns, + args.constraints, + args.getContext()); + } }, { - .source_access_type = AccessType::MYSQL, + .source_access_type = AccessType::SOURCES, }); } } - -#endif diff --git a/src/Storages/StorageExternalDistributed.h b/src/Storages/StorageExternalDistributed.h index 71022f5eaa3..a6718398a3a 100644 --- a/src/Storages/StorageExternalDistributed.h +++ b/src/Storages/StorageExternalDistributed.h @@ -4,11 +4,8 @@ #include "config_core.h" #endif -#if USE_MYSQL || USE_LIBPQXX - #include #include -#include namespace DB @@ -28,7 +25,7 @@ public: { MySQL, PostgreSQL, - Default + URL }; std::string getName() const override { return "ExternalDistributed"; } @@ -55,11 +52,19 @@ protected: const ConstraintsDescription & constraints_, ContextPtr context_); + StorageExternalDistributed( + const String & addresses_description, + const StorageID & table_id, + const String & format_name, + const std::optional & format_settings, + const String & compression_method, + const ColumnsDescription & columns, + const ConstraintsDescription & constraints, + ContextPtr context); + private: using Shards = std::unordered_set; Shards shards; }; } - -#endif diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 3a57c8ab4f6..a775ac43c29 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -79,12 +79,18 @@ StoragePtr StorageFactory::get( } else if (query.is_live_view) { - if (query.storage) throw Exception("Specifying ENGINE is not allowed for a LiveView", ErrorCodes::INCORRECT_QUERY); name = "LiveView"; } + else if (query.is_dictionary) + { + if (query.storage) + throw Exception("Specifying ENGINE is not allowed for a Dictionary", ErrorCodes::INCORRECT_QUERY); + + name = "Dictionary"; + } else { /// Check for some special types, that are not allowed to be stored in tables. Example: NULL data type. @@ -199,7 +205,7 @@ StoragePtr StorageFactory::get( assert(arguments.getContext() == arguments.getContext()->getGlobalContext()); auto res = storages.at(name).creator_fn(arguments); - if (!empty_engine_args.empty()) + if (!empty_engine_args.empty()) //-V547 { /// Storage creator modified empty arguments list, so we should modify the query assert(storage_def && storage_def->engine && !storage_def->engine->arguments); diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 8b6d7839de0..e6c2f52f925 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -1,4 +1,3 @@ -#include #include #include @@ -22,6 +21,8 @@ #include #include #include +#include +#include namespace DB @@ -29,6 +30,7 @@ namespace DB namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int NETWORK_ERROR; } IStorageURLBase::IStorageURLBase( @@ -234,6 +236,61 @@ Pipe IStorageURLBase::read( chooseCompressionMethod(request_uri.getPath(), compression_method))); } + +Pipe StorageURLWithFailover::read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + SelectQueryInfo & query_info, + ContextPtr local_context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned /*num_streams*/) +{ + auto params = getReadURIParams(column_names, metadata_snapshot, query_info, local_context, processed_stage, max_block_size); + WriteBufferFromOwnString error_message; + error_message << "Detailed description:"; + + for (const auto & uri_option : uri_options) + { + auto request_uri = uri_option; + for (const auto & [param, value] : params) + request_uri.addQueryParameter(param, value); + try + { + /// Check for uri accessibility is done in constructor of ReadWriteBufferFromHTTP while creating StorageURLSource. + auto url_source = std::make_shared( + request_uri, + getReadMethod(), + getReadPOSTDataCallback( + column_names, metadata_snapshot, query_info, + local_context, processed_stage, max_block_size), + format_name, + format_settings, + getName(), + getHeaderBlock(column_names, metadata_snapshot), + local_context, + metadata_snapshot->getColumns(), + max_block_size, + ConnectionTimeouts::getHTTPTimeouts(local_context), + chooseCompressionMethod(request_uri.getPath(), compression_method)); + + std::shuffle(uri_options.begin(), uri_options.end(), thread_local_rng); + + return Pipe(url_source); + } + catch (...) + { + error_message << " Host: " << uri_option.getHost() << ", post: " << uri_option.getPort() << ", path: " << uri_option.getPath(); + error_message << ", error: " << getCurrentExceptionMessage(false) << ";"; + + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + + throw Exception(ErrorCodes::NETWORK_ERROR, "All uri options are unreachable. {}", error_message.str()); +} + + BlockOutputStreamPtr IStorageURLBase::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) { return std::make_shared(uri, format_name, @@ -256,6 +313,64 @@ StorageURL::StorageURL(const Poco::URI & uri_, context_->getRemoteHostFilter().checkURL(uri); } + +StorageURLWithFailover::StorageURLWithFailover( + const std::vector & uri_options_, + const StorageID & table_id_, + const String & format_name_, + const std::optional & format_settings_, + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, + ContextPtr context_, + const String & compression_method_) + : StorageURL(Poco::URI(), table_id_, format_name_, format_settings_, columns_, constraints_, context_, compression_method_) +{ + for (const auto & uri_option : uri_options_) + { + Poco::URI poco_uri(uri_option); + context_->getRemoteHostFilter().checkURL(poco_uri); + uri_options.emplace_back(std::move(poco_uri)); + LOG_DEBUG(&Poco::Logger::get("StorageURLDistributed"), "Adding URL option: {}", uri_option); + } +} + + +FormatSettings StorageURL::getFormatSettingsFromArgs(const StorageFactory::Arguments & args) +{ + // Use format settings from global server context + settings from + // the SETTINGS clause of the create query. Settings from current + // session and user are ignored. + FormatSettings format_settings; + if (args.storage_def->settings) + { + FormatFactorySettings user_format_settings; + + // Apply changed settings from global context, but ignore the + // unknown ones, because we only have the format settings here. + const auto & changes = args.getContext()->getSettingsRef().changes(); + for (const auto & change : changes) + { + if (user_format_settings.has(change.name)) + { + user_format_settings.set(change.name, change.value); + } + } + + // Apply changes from SETTINGS clause, with validation. + user_format_settings.applyChanges(args.storage_def->settings->changes); + + format_settings = getFormatSettings(args.getContext(), + user_format_settings); + } + else + { + format_settings = getFormatSettings(args.getContext()); + } + + return format_settings; +} + + void registerStorageURL(StorageFactory & factory) { factory.registerStorage("URL", [](const StorageFactory::Arguments & args) @@ -268,53 +383,21 @@ void registerStorageURL(StorageFactory & factory) engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[0], args.getLocalContext()); - String url = engine_args[0]->as().value.safeGet(); + const String & url = engine_args[0]->as().value.safeGet(); Poco::URI uri(url); engine_args[1] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[1], args.getLocalContext()); - String format_name = engine_args[1]->as().value.safeGet(); + const String & format_name = engine_args[1]->as().value.safeGet(); - String compression_method; + String compression_method = "auto"; if (engine_args.size() == 3) { engine_args[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[2], args.getLocalContext()); compression_method = engine_args[2]->as().value.safeGet(); } - else - { - compression_method = "auto"; - } - // Use format settings from global server context + settings from - // the SETTINGS clause of the create query. Settings from current - // session and user are ignored. - FormatSettings format_settings; - if (args.storage_def->settings) - { - FormatFactorySettings user_format_settings; - - // Apply changed settings from global context, but ignore the - // unknown ones, because we only have the format settings here. - const auto & changes = args.getContext()->getSettingsRef().changes(); - for (const auto & change : changes) - { - if (user_format_settings.has(change.name)) - { - user_format_settings.set(change.name, change.value); - } - } - - // Apply changes from SETTINGS clause, with validation. - user_format_settings.applyChanges(args.storage_def->settings->changes); - - format_settings = getFormatSettings(args.getContext(), - user_format_settings); - } - else - { - format_settings = getFormatSettings(args.getContext()); - } + auto format_settings = StorageURL::getFormatSettingsFromArgs(args); return StorageURL::create( uri, diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 6fc1a6006ec..012915c9b24 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -53,7 +54,6 @@ protected: // In this case, format_settings is not set. std::optional format_settings; -private: virtual std::string getReadMethod() const; virtual std::vector> getReadURIParams( @@ -72,6 +72,7 @@ private: QueryProcessingStage::Enum & processed_stage, size_t max_block_size) const; +private: virtual Block getHeaderBlock(const Names & column_names, const StorageMetadataPtr & metadata_snapshot) const = 0; }; @@ -102,7 +103,7 @@ private: BlockOutputStreamPtr writer; }; -class StorageURL final : public ext::shared_ptr_helper, public IStorageURLBase +class StorageURL : public ext::shared_ptr_helper, public IStorageURLBase { friend struct ext::shared_ptr_helper; public: @@ -124,5 +125,35 @@ public: { return metadata_snapshot->getSampleBlock(); } + + static FormatSettings getFormatSettingsFromArgs(const StorageFactory::Arguments & args); +}; + + +/// StorageURLWithFailover is allowed only for URL table function, not as a separate storage. +class StorageURLWithFailover final : public StorageURL +{ +public: + StorageURLWithFailover( + const std::vector & uri_options_, + const StorageID & table_id_, + const String & format_name_, + const std::optional & format_settings_, + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, + ContextPtr context_, + const String & compression_method_); + + Pipe read( + const Names & column_names, + const StorageMetadataPtr & /*metadata_snapshot*/, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + +private: + std::vector uri_options; }; } diff --git a/src/Storages/System/StorageSystemDictionaries.cpp b/src/Storages/System/StorageSystemDictionaries.cpp index c76dba9df58..5f4d5df2036 100644 --- a/src/Storages/System/StorageSystemDictionaries.cpp +++ b/src/Storages/System/StorageSystemDictionaries.cpp @@ -60,9 +60,13 @@ NamesAndTypesList StorageSystemDictionaries::getVirtuals() const void StorageSystemDictionaries::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo & /*query_info*/) const { const auto access = context->getAccess(); - const bool check_access_for_dictionaries = !access->isGranted(AccessType::SHOW_DICTIONARIES); + const bool check_access_for_dictionaries = access->isGranted(AccessType::SHOW_DICTIONARIES); const auto & external_dictionaries = context->getExternalDictionariesLoader(); + + if (!check_access_for_dictionaries) + return; + for (const auto & load_result : external_dictionaries.getLoadResults()) { const auto dict_ptr = std::dynamic_pointer_cast(load_result.object); @@ -77,8 +81,7 @@ void StorageSystemDictionaries::fillData(MutableColumns & res_columns, ContextPt dict_id.table_name = load_result.name; String db_or_tag = dict_id.database_name.empty() ? IDictionary::NO_DATABASE_TAG : dict_id.database_name; - if (check_access_for_dictionaries - && !access->isGranted(AccessType::SHOW_DICTIONARIES, db_or_tag, dict_id.table_name)) + if (!access->isGranted(AccessType::SHOW_DICTIONARIES, db_or_tag, dict_id.table_name)) continue; size_t i = 0; diff --git a/src/Storages/tests/CMakeLists.txt b/src/Storages/examples/CMakeLists.txt similarity index 100% rename from src/Storages/tests/CMakeLists.txt rename to src/Storages/examples/CMakeLists.txt diff --git a/src/Storages/tests/active_parts.py b/src/Storages/examples/active_parts.py similarity index 100% rename from src/Storages/tests/active_parts.py rename to src/Storages/examples/active_parts.py diff --git a/src/Storages/tests/columns_description_fuzzer.cpp b/src/Storages/examples/columns_description_fuzzer.cpp similarity index 100% rename from src/Storages/tests/columns_description_fuzzer.cpp rename to src/Storages/examples/columns_description_fuzzer.cpp diff --git a/src/Storages/tests/get_abandonable_lock_in_all_partitions.cpp b/src/Storages/examples/get_abandonable_lock_in_all_partitions.cpp similarity index 100% rename from src/Storages/tests/get_abandonable_lock_in_all_partitions.cpp rename to src/Storages/examples/get_abandonable_lock_in_all_partitions.cpp diff --git a/src/Storages/tests/get_current_inserts_in_replicated.cpp b/src/Storages/examples/get_current_inserts_in_replicated.cpp similarity index 100% rename from src/Storages/tests/get_current_inserts_in_replicated.cpp rename to src/Storages/examples/get_current_inserts_in_replicated.cpp diff --git a/src/Storages/tests/merge_selector.cpp b/src/Storages/examples/merge_selector.cpp similarity index 100% rename from src/Storages/tests/merge_selector.cpp rename to src/Storages/examples/merge_selector.cpp diff --git a/src/Storages/tests/merge_selector2.cpp b/src/Storages/examples/merge_selector2.cpp similarity index 100% rename from src/Storages/tests/merge_selector2.cpp rename to src/Storages/examples/merge_selector2.cpp diff --git a/src/Storages/tests/mergetree_checksum_fuzzer.cpp b/src/Storages/examples/mergetree_checksum_fuzzer.cpp similarity index 100% rename from src/Storages/tests/mergetree_checksum_fuzzer.cpp rename to src/Storages/examples/mergetree_checksum_fuzzer.cpp diff --git a/src/Storages/tests/remove_symlink_directory.cpp b/src/Storages/examples/remove_symlink_directory.cpp similarity index 100% rename from src/Storages/tests/remove_symlink_directory.cpp rename to src/Storages/examples/remove_symlink_directory.cpp diff --git a/src/Storages/tests/transform_part_zk_nodes.cpp b/src/Storages/examples/transform_part_zk_nodes.cpp similarity index 100% rename from src/Storages/tests/transform_part_zk_nodes.cpp rename to src/Storages/examples/transform_part_zk_nodes.cpp diff --git a/src/TableFunctions/TableFunctionDictionary.cpp b/src/TableFunctions/TableFunctionDictionary.cpp index 46d3183bba9..268f49b912e 100644 --- a/src/TableFunctions/TableFunctionDictionary.cpp +++ b/src/TableFunctions/TableFunctionDictionary.cpp @@ -52,7 +52,15 @@ StoragePtr TableFunctionDictionary::executeImpl( { StorageID dict_id(getDatabaseName(), table_name); auto dictionary_table_structure = getActualTableStructure(context); - return StorageDictionary::create(dict_id, dictionary_name, std::move(dictionary_table_structure), StorageDictionary::Location::Custom); + + auto result = StorageDictionary::create( + dict_id, + dictionary_name, + std::move(dictionary_table_structure), + StorageDictionary::Location::Custom, + context); + + return result; } void registerTableFunctionDictionary(TableFunctionFactory & factory) diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index a77b9140508..c2acb3ee207 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -15,10 +16,26 @@ StoragePtr TableFunctionURL::getStorage( const String & source, const String & format_, const ColumnsDescription & columns, ContextPtr global_context, const std::string & table_name, const String & compression_method_) const { - Poco::URI uri(source); - return StorageURL::create(uri, StorageID(getDatabaseName(), table_name), - format_, std::nullopt /*format settings*/, columns, - ConstraintsDescription{}, global_context, compression_method_); + /// If url contains {1..k} or failover options with separator `|`, use a separate storage + if ((source.find('{') == std::string::npos || source.find('}') == std::string::npos) && source.find('|') == std::string::npos) + { + Poco::URI uri(source); + return StorageURL::create(uri, StorageID(getDatabaseName(), table_name), + format_, std::nullopt /*format settings*/, columns, + ConstraintsDescription{}, global_context, compression_method_); + } + else + { + return StorageExternalDistributed::create( + source, + StorageID(getDatabaseName(), table_name), + format_, + std::nullopt, + compression_method_, + columns, + ConstraintsDescription{}, + global_context); + } } void registerTableFunctionURL(TableFunctionFactory & factory) diff --git a/tests/integration/helpers/hdfs_api.py b/tests/integration/helpers/hdfs_api.py index 5b1b4c402da..a77d7cf4c7a 100644 --- a/tests/integration/helpers/hdfs_api.py +++ b/tests/integration/helpers/hdfs_api.py @@ -60,6 +60,9 @@ class HDFSApi(object): #principal=self.principal, #hostname_override=self.host, principal=self.principal) # , mutual_authentication=reqkerb.REQUIRED, force_preemptive=True) + self.kerberos_auth = reqkerb.HTTPKerberosAuth(mutual_authentication=reqkerb.DISABLED, hostname_override=self.host, principal=self.principal) + if self.kerberos_auth is None: + print("failed to obtain kerberos_auth") else: self.kerberos_auth = None @@ -94,23 +97,24 @@ class HDFSApi(object): raise Exception("Kinit running failure") - def read_data(self, path, universal_newlines=True): - logging.debug("read_data protocol:{} host:{} port:{} path: {}".format(self.protocol, self.host, self.proxy_port, path)) - response = requests.get("{protocol}://{host}:{port}/webhdfs/v1{path}?op=OPEN".format(protocol=self.protocol, host=self.host, port=self.proxy_port, path=path), headers={'host': 'localhost'}, allow_redirects=False, verify=False, auth=self.kerberos_auth) - if response.status_code != 307: - response.raise_for_status() - # additional_params = '&'.join(response.headers['Location'].split('&')[1:2]) - location = None - if self.kerberized: - location = response.headers['Location'].replace("kerberizedhdfs1:1006", "{}:{}".format(self.host, self.data_port)) - else: - location = response.headers['Location'].replace("hdfs1:50075", "{}:{}".format(self.host, self.data_port)) - logging.debug("redirected to {}".format(location)) + def req_wrapper(self, func, expected_code, cnt=2, **kwargs): + with dns_hook(self): + for i in range(0, cnt): + response_data = func(**kwargs) + if response_data.status_code == expected_code: + return response_data + else: + print("unexpected response_data.status_code {}", response_data.status_code) + response_data.raise_for_status() - response_data = requests.get(location, headers={'host': 'localhost'}, - verify=False, auth=self.kerberos_auth) - if response_data.status_code != 200: - response_data.raise_for_status() + def read_data(self, path, universal_newlines=True): + response = self.req_wrapper(requests.get, 307, url="{protocol}://{host}:{port}/webhdfs/v1{path}?op=OPEN".format(protocol=self.protocol, host=self.host, port=self.proxy_port, path=path), headers={'host': 'localhost'}, allow_redirects=False, verify=False, auth=self.kerberos_auth) + # additional_params = '&'.join(response.headers['Location'].split('&')[1:2]) + url = "{location}".format(location=response.headers['Location']) + # print("redirected to ", url) + response_data = self.req_wrapper(requests.get, 200, url=url, + headers={'host': 'localhost'}, + verify=False, auth=self.kerberos_auth) if universal_newlines: return response_data.text else: @@ -126,41 +130,37 @@ class HDFSApi(object): named_file.write(content) named_file.flush() - response = requests.put( - "{protocol}://{host}:{port}/webhdfs/v1{path}?op=CREATE".format(protocol=self.protocol, host='localhost', - port=self.proxy_port, - path=path, user=self.user), - allow_redirects=False, - headers={'host': 'localhost'}, - params={'overwrite' : 'true'}, - verify=False, auth=self.kerberos_auth - ) - logging.debug("HDFS api response:{}".format(response.headers)) - - if response.status_code != 307: - response.raise_for_status() - - # additional_params = '&'.join( - # response.headers['Location'].split('&')[1:2] + ["user.name={}".format(self.user), "overwrite=true"]) if self.kerberized: - location = response.headers['Location'].replace("kerberizedhdfs1:1006", "{}:{}".format(self.host, self.data_port)) - else: - location = response.headers['Location'].replace("hdfs1:50075", "{}:{}".format(self.host, self.data_port)) - + self._run_kinit() + self.kerberos_auth = reqkerb.HTTPKerberosAuth(mutual_authentication=reqkerb.DISABLED, hostname_override=self.host, principal=self.principal) + # print(self.kerberos_auth) + + response = self.req_wrapper(requests.put, 307, + url="{protocol}://{host}:{port}/webhdfs/v1{path}?op=CREATE".format( + protocol=self.protocol, host=self.host, + port=self.proxy_port, + path=path, user=self.user), + allow_redirects=False, + headers={'host': 'localhost'}, + params={'overwrite' : 'true'}, + verify=False, auth=self.kerberos_auth + ) + additional_params = '&'.join( + response.headers['Location'].split('&')[1:2] + ["user.name={}".format(self.user), "overwrite=true"]) + with open(fpath, mode="rb") as fh: file_data = fh.read() protocol = "http" # self.protocol - response = requests.put( - "{location}".format(location=location), - data=file_data, - headers={'content-type':'text/plain', 'host': 'localhost'}, - params={'file': path, 'user.name' : self.user}, - allow_redirects=False, verify=False, auth=self.kerberos_auth + response = self.req_wrapper(requests.put, 201, + url="{location}".format( + location=response.headers['Location']), + data=file_data, + headers={'content-type':'text/plain', 'host': 'localhost'}, + params={'file': path, 'user.name' : self.user}, + allow_redirects=False, verify=False, auth=self.kerberos_auth ) - logging.debug(response) - if response.status_code != 201: - response.raise_for_status() + # print(response) def write_gzip_data(self, path, content): diff --git a/tests/integration/test_dictionaries_ddl/test.py b/tests/integration/test_dictionaries_ddl/test.py index ea0de044e6a..84374fde246 100644 --- a/tests/integration/test_dictionaries_ddl/test.py +++ b/tests/integration/test_dictionaries_ddl/test.py @@ -292,6 +292,23 @@ def test_clickhouse_remote(started_cluster): time.sleep(0.5) node3.query("detach dictionary if exists test.clickhouse_remote") + + with pytest.raises(QueryRuntimeException): + node3.query(""" + CREATE DICTIONARY test.clickhouse_remote( + id UInt64, + SomeValue1 UInt8, + SomeValue2 String + ) + PRIMARY KEY id + LAYOUT(FLAT()) + SOURCE(CLICKHOUSE(HOST 'node4' PORT 9000 USER 'default' PASSWORD 'default' TABLE 'xml_dictionary_table' DB 'test')) + LIFETIME(MIN 1 MAX 10) + """) + + node3.query("attach dictionary test.clickhouse_remote") + node3.query("drop dictionary test.clickhouse_remote") + node3.query(""" CREATE DICTIONARY test.clickhouse_remote( id UInt64, diff --git a/tests/integration/test_library_bridge/test.py b/tests/integration/test_library_bridge/test.py index f0aeb85a52b..422a7ee954f 100644 --- a/tests/integration/test_library_bridge/test.py +++ b/tests/integration/test_library_bridge/test.py @@ -43,6 +43,9 @@ def setup_teardown(): def test_load_all(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + instance.query(''' CREATE DICTIONARY lib_dict (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key @@ -81,6 +84,9 @@ def test_load_all(ch_cluster): def test_load_ids(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + instance.query(''' CREATE DICTIONARY lib_dict_c (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key SOURCE(library(PATH '/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so')) @@ -101,6 +107,9 @@ def test_load_ids(ch_cluster): def test_load_keys(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + instance.query(''' CREATE DICTIONARY lib_dict_ckc (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key @@ -117,6 +126,9 @@ def test_load_keys(ch_cluster): def test_load_all_many_rows(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + num_rows = [1000, 10000, 100000, 1000000] for num in num_rows: instance.query(''' @@ -136,6 +148,9 @@ def test_load_all_many_rows(ch_cluster): def test_null_values(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + instance.query('SYSTEM RELOAD DICTIONARY dict2') instance.query(""" CREATE TABLE IF NOT EXISTS `dict2_table` ( diff --git a/tests/integration/test_redirect_url_storage/test.py b/tests/integration/test_redirect_url_storage/test.py index 736fb5c409c..d51b7e3287e 100644 --- a/tests/integration/test_redirect_url_storage/test.py +++ b/tests/integration/test_redirect_url_storage/test.py @@ -28,6 +28,32 @@ def test_url_without_redirect(started_cluster): assert node1.query("select * from WebHDFSStorage") == "1\tMark\t72.53\n" +def test_url_with_globs(started_cluster): + started_cluster.hdfs_api.write_data("/simple_storage_1_1", "1\n") + started_cluster.hdfs_api.write_data("/simple_storage_1_2", "2\n") + started_cluster.hdfs_api.write_data("/simple_storage_1_3", "3\n") + started_cluster.hdfs_api.write_data("/simple_storage_2_1", "4\n") + started_cluster.hdfs_api.write_data("/simple_storage_2_2", "5\n") + started_cluster.hdfs_api.write_data("/simple_storage_2_3", "6\n") + + result = node1.query( + "select * from url('http://hdfs1:50075/webhdfs/v1/simple_storage_{1..2}_{1..3}?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'data String') as data order by data") + assert result == "1\n2\n3\n4\n5\n6\n" + + +def test_url_with_globs_and_failover(started_cluster): + started_cluster.hdfs_api.write_data("/simple_storage_1_1", "1\n") + started_cluster.hdfs_api.write_data("/simple_storage_1_2", "2\n") + started_cluster.hdfs_api.write_data("/simple_storage_1_3", "3\n") + started_cluster.hdfs_api.write_data("/simple_storage_3_1", "4\n") + started_cluster.hdfs_api.write_data("/simple_storage_3_2", "5\n") + started_cluster.hdfs_api.write_data("/simple_storage_3_3", "6\n") + + result = node1.query( + "select * from url('http://hdfs1:50075/webhdfs/v1/simple_storage_{0|1|2|3}_{1..3}?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'data String') as data order by data") + assert result == "1\n2\n3\n" + + def test_url_with_redirect_not_allowed(started_cluster): hdfs_api = started_cluster.make_hdfs_api() diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference b/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference index e591300eddc..a4e2f380eb8 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_create.reference @@ -12,7 +12,10 @@ db_01018 dict1 ==DROP DICTIONARY 0 =DICTIONARY in Memory DB -0 +CREATE DICTIONARY memory_db.dict2\n(\n `key_column` UInt64 DEFAULT 0 INJECTIVE,\n `second_column` UInt8 DEFAULT 1 EXPRESSION rand() % 222,\n `third_column` String DEFAULT \'qqq\'\n)\nPRIMARY KEY key_column\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT tcpPort() USER \'default\' TABLE \'table_for_dict\' PASSWORD \'\' DB \'database_for_dict_01018\'))\nLIFETIME(MIN 1 MAX 10)\nLAYOUT(FLAT()) +dict2 +1 +memory_db dict2 =DICTIONARY in Lazy DB =DROP DATABASE WITH DICTIONARY dict4 diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql b/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql index f7d34071eb0..dd62f1451a8 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql @@ -89,9 +89,9 @@ CREATE DICTIONARY memory_db.dict2 PRIMARY KEY key_column SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict_01018')) LIFETIME(MIN 1 MAX 10) -LAYOUT(FLAT()); -- {serverError 48} +LAYOUT(FLAT()); -SHOW CREATE DICTIONARY memory_db.dict2; -- {serverError 487} +SHOW CREATE DICTIONARY memory_db.dict2; SHOW DICTIONARIES FROM memory_db LIKE 'dict2'; @@ -114,7 +114,7 @@ CREATE DICTIONARY lazy_db.dict3 PRIMARY KEY key_column, second_column SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict_01018')) LIFETIME(MIN 1 MAX 10) -LAYOUT(COMPLEX_KEY_HASHED()); -- {serverError 48} +LAYOUT(COMPLEX_KEY_HASHED()); --{serverError 1} DROP DATABASE IF EXISTS lazy_db; diff --git a/tests/queries/0_stateless/01190_full_attach_syntax.sql b/tests/queries/0_stateless/01190_full_attach_syntax.sql index 62e77f9870e..eb739782e5f 100644 --- a/tests/queries/0_stateless/01190_full_attach_syntax.sql +++ b/tests/queries/0_stateless/01190_full_attach_syntax.sql @@ -18,7 +18,7 @@ SHOW CREATE DICTIONARY test_01190.dict; CREATE TABLE log ENGINE = Log AS SELECT 'test' AS s; SHOW CREATE log; DETACH TABLE log; -ATTACH DICTIONARY log; -- { serverError 487 } +ATTACH DICTIONARY log; -- { serverError 80 } ATTACH TABLE log (s String) ENGINE = Log(); SHOW CREATE log; SELECT * FROM log; diff --git a/tests/queries/0_stateless/01191_rename_dictionary.sql b/tests/queries/0_stateless/01191_rename_dictionary.sql index 3656d49f6e6..264c527ccca 100644 --- a/tests/queries/0_stateless/01191_rename_dictionary.sql +++ b/tests/queries/0_stateless/01191_rename_dictionary.sql @@ -13,11 +13,15 @@ INSERT INTO test_01191._ VALUES (42, 'test'); SELECT name, status FROM system.dictionaries WHERE database='test_01191'; SELECT name, engine FROM system.tables WHERE database='test_01191' ORDER BY name; -RENAME DICTIONARY test_01191.table TO test_01191.table1; -- {serverError 80} -EXCHANGE TABLES test_01191.table AND test_01191.dict; -- {serverError 48} +RENAME DICTIONARY test_01191.table TO test_01191.table1; -- {serverError 60} +EXCHANGE TABLES test_01191.table AND test_01191.dict; -- {serverError 60} EXCHANGE TABLES test_01191.dict AND test_01191.table; -- {serverError 80} RENAME TABLE test_01191.dict TO test_01191.dict1; -- {serverError 80} -RENAME DICTIONARY test_01191.dict TO default.dict1; -- {serverError 48} + +CREATE DATABASE dummy_db ENGINE=Atomic; +RENAME DICTIONARY test_01191.dict TO dummy_db.dict1; +RENAME DICTIONARY dummy_db.dict1 TO test_01191.dict; +DROP DATABASE dummy_db; RENAME DICTIONARY test_01191.dict TO test_01191.dict1; diff --git a/tests/queries/0_stateless/01560_mann_whitney.sql b/tests/queries/0_stateless/01560_mann_whitney.sql index 6e1ac55349d..e3a9b4ecd03 100644 --- a/tests/queries/0_stateless/01560_mann_whitney.sql +++ b/tests/queries/0_stateless/01560_mann_whitney.sql @@ -6,4 +6,5 @@ SELECT '223.0', '0.5426959774289482'; WITH mannWhitneyUTest(left, right) AS pair SELECT roundBankers(pair.1, 16) as t_stat, roundBankers(pair.2, 16) as p_value from mann_whitney_test; WITH mannWhitneyUTest('two-sided', 1)(left, right) as pair SELECT roundBankers(pair.1, 16) as t_stat, roundBankers(pair.2, 16) as p_value from mann_whitney_test; WITH mannWhitneyUTest('two-sided')(left, right) as pair SELECT roundBankers(pair.1, 16) as t_stat, roundBankers(pair.2, 16) as p_value from mann_whitney_test; +WITH mannWhitneyUTest('two-sided')(1, right) AS pair SELECT roundBankers(pair.1, 16) AS t_stat, roundBankers(pair.2, 16) AS p_value FROM mann_whitney_test; --{serverError 36} DROP TABLE IF EXISTS mann_whitney_test; diff --git a/tests/queries/0_stateless/01786_explain_merge_tree.reference b/tests/queries/0_stateless/01786_explain_merge_tree.reference index 51eb52688a3..7a0a0af3e05 100644 --- a/tests/queries/0_stateless/01786_explain_merge_tree.reference +++ b/tests/queries/0_stateless/01786_explain_merge_tree.reference @@ -30,6 +30,64 @@ Description: set GRANULARITY 2 Parts: 1/1 Granules: 1/2 +----------------- + "Node Type": "ReadFromMergeTree", + "Indexes": [ + { + "Type": "MinMax", + "Keys": ["y"], + "Condition": "(y in [1, +inf))", + "Initial Parts": 5, + "Selected Parts": 4, + "Initial Granules": 12, + "Selected Granules": 11 + }, + { + "Type": "Partition", + "Keys": ["y", "bitAnd(z, 3)"], + "Condition": "and((bitAnd(z, 3) not in [1, 1]), and((y in [1, +inf)), (bitAnd(z, 3) not in [1, 1])))", + "Initial Parts": 4, + "Selected Parts": 3, + "Initial Granules": 11, + "Selected Granules": 10 + }, + { + "Type": "PrimaryKey", + "Keys": ["x", "y"], + "Condition": "and((x in [11, +inf)), (y in [1, +inf)))", + "Initial Parts": 3, + "Selected Parts": 2, + "Initial Granules": 10, + "Selected Granules": 6 + }, + { + "Type": "Skip", + "Name": "t_minmax", + "Description": "minmax GRANULARITY 2", + "Initial Parts": 2, + "Selected Parts": 1, + "Initial Granules": 6, + "Selected Granules": 2 + }, + { + "Type": "Skip", + "Name": "t_set", + "Description": "set GRANULARITY 2", + "Initial Parts": 1, + "Selected Parts": 1, + "Initial Granules": 2, + "Selected Granules": 1 + } + ] + } + ] + } + ] + } + ] + } + } +] ----------------- ReadFromMergeTree ReadType: InOrder diff --git a/tests/queries/0_stateless/01786_explain_merge_tree.sh b/tests/queries/0_stateless/01786_explain_merge_tree.sh index 2791d0c6921..6be86f9ce02 100755 --- a/tests/queries/0_stateless/01786_explain_merge_tree.sh +++ b/tests/queries/0_stateless/01786_explain_merge_tree.sh @@ -16,6 +16,12 @@ $CLICKHOUSE_CLIENT -q " echo "-----------------" +$CLICKHOUSE_CLIENT -q " + explain indexes = 1, json = 1 select *, _part from test_index where t % 19 = 16 and y > 0 and bitAnd(z, 3) != 1 and x > 10 and t % 20 > 14 format TSVRaw; + " | grep -A 100 "ReadFromMergeTree" # | grep -v "Description" + +echo "-----------------" + $CLICKHOUSE_CLIENT -q " explain actions = 1 select x from test_index where x > 15 order by x; " | grep -A 100 "ReadFromMergeTree" diff --git a/tests/queries/0_stateless/01802_rank_corr_mann_whitney_over_window.reference b/tests/queries/0_stateless/01802_rank_corr_mann_whitney_over_window.reference index 42acbe4fbaf..e69de29bb2d 100644 --- a/tests/queries/0_stateless/01802_rank_corr_mann_whitney_over_window.reference +++ b/tests/queries/0_stateless/01802_rank_corr_mann_whitney_over_window.reference @@ -1,10 +0,0 @@ -0.5060606060606061 -0.5083333333333333 -0.5119047619047619 -0.5178571428571428 -0.5285714285714286 -0.525 -0.55 -0.625 -0.5 -nan diff --git a/tests/queries/0_stateless/01802_rank_corr_mann_whitney_over_window.sql b/tests/queries/0_stateless/01802_rank_corr_mann_whitney_over_window.sql index 3c1746a30f8..24ee9282ac0 100644 --- a/tests/queries/0_stateless/01802_rank_corr_mann_whitney_over_window.sql +++ b/tests/queries/0_stateless/01802_rank_corr_mann_whitney_over_window.sql @@ -13,10 +13,10 @@ ENGINE = MergeTree ORDER BY enroll_date SETTINGS index_granularity = 8192; -INSERT INTO 01802_empsalary VALUES ('sales', 1, 5000, '2006-10-01'), ('develop', 8, 6000, '2006-10-01'), ('personnel', 2, 3900, '2006-12-23'), ('develop', 10, 5200, '2007-08-01'), ('sales', 3, 4800, '2007-08-01'), ('sales', 4, 4800, '2007-08-08'), ('develop', 11, 5200, '2007-08-15'), ('personnel', 5, 3500, '2007-12-10'), ('develop', 7, 4200, '2008-01-01'), ('develop', 9, 4500, '2008-01-01'); +INSERT INTO 01802_empsalary VALUES ('sales', 1, 5000, '2006-10-01'), ('develop', 8, 6000, '2006-10-01'), ('personnel', 2, 3900, '2006-12-23'), ('develop', 10, 5200, '2007-08-01'), ('sales', 3, 4800, '2007-08-01'), ('sales', 4, 4801, '2007-08-08'), ('develop', 11, 5200, '2007-08-15'), ('personnel', 5, 3500, '2007-12-10'), ('develop', 7, 4200, '2008-01-01'), ('develop', 9, 4500, '2008-01-01'); SELECT mannWhitneyUTest(salary, salary) OVER (ORDER BY salary ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS func FROM 01802_empsalary; -- {serverError 36} -SELECT rankCorr(salary, 0.5) OVER (ORDER BY salary ASC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS func FROM 01802_empsalary; +SELECT rankCorr(salary, 0.5) OVER (ORDER BY salary ASC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS func FROM 01802_empsalary; -- {serverError 36} DROP TABLE IF EXISTS 01802_empsalary; diff --git a/tests/queries/0_stateless/01823_explain_json.reference b/tests/queries/0_stateless/01823_explain_json.reference new file mode 100644 index 00000000000..5c7845a22d5 --- /dev/null +++ b/tests/queries/0_stateless/01823_explain_json.reference @@ -0,0 +1,141 @@ +[ + { + "Plan": { + "Node Type": "Union", + "Plans": [ + { + "Node Type": "Expression", + "Plans": [ + { + "Node Type": "SettingQuotaAndLimits", + "Plans": [ + { + "Node Type": "ReadFromStorage" + } + ] + } + ] + }, + { + "Node Type": "Expression", + "Plans": [ + { + "Node Type": "SettingQuotaAndLimits", + "Plans": [ + { + "Node Type": "ReadFromStorage" + } + ] + } + ] + } + ] + } + } +] +-------- + "Header": [ + { + "Name": "1", + "Type": "UInt8" + }, + { + "Name": "plus(2, dummy)", + "Type": "UInt16" + } +-------- + "Node Type": "Aggregating", + "Header": [ + { + "Name": "number", + "Type": "UInt64" + }, + { + "Name": "plus(number, 1)", + "Type": "UInt64" + }, + { + "Name": "quantile(0.2)(number)", + "Type": "Float64" + }, + { + "Name": "sumIf(number, greater(number, 0))", + "Type": "UInt64" + } + ], + "Keys": ["number", "plus(number, 1)"], + "Aggregates": [ + { + "Name": "quantile(0.2)(number)", + "Function": { + "Name": "quantile", + "Parameters": ["0.2"], + "Argument Types": ["UInt64"], + "Result Type": "Float64" + }, + "Arguments": ["number"], + "Argument Positions": [0] + }, + { + "Name": "sumIf(number, greater(number, 0))", + "Function": { + "Name": "sumIf", + "Argument Types": ["UInt64", "UInt8"], + "Result Type": "UInt64" + }, + "Arguments": ["number", "greater(number, 0)"], + "Argument Positions": [0, 2] + } + ], +-------- + "Node Type": "ArrayJoin", + "Left": false, + "Columns": ["x", "y"], +-------- + "Node Type": "Distinct", + "Columns": ["intDiv(number, 3)", "intDiv(number, 2)"], +-- + "Node Type": "Distinct", + "Columns": ["intDiv(number, 3)", "intDiv(number, 2)"], +-------- + "Sort Description": [ + { + "Column": "number", + "Ascending": false, + "With Fill": false + }, + { + "Column": "plus(number, 1)", + "Ascending": true, + "With Fill": false + } + ], + "Limit": 3, +-- + "Sort Description": [ + { + "Column": "number", + "Ascending": false, + "With Fill": false + }, + { + "Column": "plus(number, 1)", + "Ascending": true, + "With Fill": false + } + ], + "Limit": 3, +-- + "Sort Description": [ + { + "Column": "number", + "Ascending": false, + "With Fill": false + }, + { + "Column": "plus(number, 1)", + "Ascending": true, + "With Fill": false + } + ], + "Limit": 3, diff --git a/tests/queries/0_stateless/01823_explain_json.sh b/tests/queries/0_stateless/01823_explain_json.sh new file mode 100755 index 00000000000..004e85cd965 --- /dev/null +++ b/tests/queries/0_stateless/01823_explain_json.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "EXPLAIN json = 1, description = 0 SELECT 1 UNION ALL SELECT 2 FORMAT TSVRaw" +echo "--------" +$CLICKHOUSE_CLIENT -q "explain json = 1, description = 0, header = 1 select 1, 2 + dummy FORMAT TSVRaw" | grep Header -m 1 -A 8 + +echo "--------" +$CLICKHOUSE_CLIENT -q "EXPLAIN json = 1, actions = 1, header = 1, description = 0 + SELECT quantile(0.2)(number), sumIf(number, number > 0) from numbers(2) group by number, number + 1 FORMAT TSVRaw + " | grep Aggregating -A 42 + +echo "--------" +$CLICKHOUSE_CLIENT -q "EXPLAIN json = 1, actions = 1, description = 0 + SELECT x, y from numbers(2) array join [number, 1] as x, [number + 1] as y FORMAT TSVRaw + " | grep ArrayJoin -A 2 + +echo "--------" +$CLICKHOUSE_CLIENT -q "EXPLAIN json = 1, actions = 1, description = 0 + SELECT distinct intDiv(number, 2), intDiv(number, 3) from numbers(10) FORMAT TSVRaw + " | grep Distinct -A 1 + +echo "--------" +$CLICKHOUSE_CLIENT -q "EXPLAIN json = 1, actions = 1, description = 0 + SELECT number + 1 from numbers(10) order by number desc, number + 1 limit 3 FORMAT TSVRaw + " | grep "Sort Description" -A 12 diff --git a/tests/queries/0_stateless/01837_database_memory_ddl_dictionaries.reference b/tests/queries/0_stateless/01837_database_memory_ddl_dictionaries.reference new file mode 100644 index 00000000000..71c4a605031 --- /dev/null +++ b/tests/queries/0_stateless/01837_database_memory_ddl_dictionaries.reference @@ -0,0 +1,3 @@ +1 First +2 Second +3 Third diff --git a/tests/queries/0_stateless/01837_database_memory_ddl_dictionaries.sql b/tests/queries/0_stateless/01837_database_memory_ddl_dictionaries.sql new file mode 100644 index 00000000000..baba8b5188e --- /dev/null +++ b/tests/queries/0_stateless/01837_database_memory_ddl_dictionaries.sql @@ -0,0 +1,30 @@ +DROP DATABASE IF EXISTS 01837_db; +CREATE DATABASE 01837_db ENGINE = Memory; + +DROP TABLE IF EXISTS 01837_db.simple_key_dictionary_source; +CREATE TABLE 01837_db.simple_key_dictionary_source +( + id UInt64, + value String +) ENGINE = TinyLog; + +INSERT INTO 01837_db.simple_key_dictionary_source VALUES (1, 'First'); +INSERT INTO 01837_db.simple_key_dictionary_source VALUES (2, 'Second'); +INSERT INTO 01837_db.simple_key_dictionary_source VALUES (3, 'Third'); + +DROP DICTIONARY IF EXISTS 01837_db.simple_key_direct_dictionary; +CREATE DICTIONARY 01837_db.simple_key_direct_dictionary +( + id UInt64, + value String +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() DB '01837_db' TABLE 'simple_key_dictionary_source')) +LAYOUT(DIRECT()); + +SELECT * FROM 01837_db.simple_key_direct_dictionary; + +DROP DICTIONARY 01837_db.simple_key_direct_dictionary; +DROP TABLE 01837_db.simple_key_dictionary_source; + +DROP DATABASE 01837_db; diff --git a/tests/queries/0_stateless/01848_http_insert_segfault.reference b/tests/queries/0_stateless/01848_http_insert_segfault.reference new file mode 100644 index 00000000000..587579af915 --- /dev/null +++ b/tests/queries/0_stateless/01848_http_insert_segfault.reference @@ -0,0 +1 @@ +Ok. diff --git a/tests/queries/0_stateless/01848_http_insert_segfault.sh b/tests/queries/0_stateless/01848_http_insert_segfault.sh new file mode 100755 index 00000000000..a263ded44eb --- /dev/null +++ b/tests/queries/0_stateless/01848_http_insert_segfault.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + + CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + # shellcheck source=../shell_config.sh + . "$CUR_DIR"/../shell_config.sh + + ${CLICKHOUSE_LOCAL} -q "select col1, initializeAggregation('argMaxState', col2, insertTime) as col2, now() as insertTime FROM generateRandom('col1 String, col2 Array(Float64)') LIMIT 1000000 FORMAT CSV" | curl -s 'http://localhost:8123/?query=INSERT%20INTO%20non_existing_table%20SELECT%20col1%2C%20initializeAggregation(%27argMaxState%27%2C%20col2%2C%20insertTime)%20as%20col2%2C%20now()%20as%20insertTime%20FROM%20input(%27col1%20String%2C%20col2%20Array(Float64)%27)%20FORMAT%20CSV' --data-binary @- | grep -q "Table default.non_existing_table doesn't exist" && echo 'Ok.' || echo 'FAIL' ||: +