From 8b82ff215fe4b42216aa41adf9907aba37a3817d Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Sat, 27 Jun 2020 04:08:21 +0300 Subject: [PATCH 01/24] muzzy decay 10 ms --- contrib/jemalloc-cmake/CMakeLists.txt | 7 ++++++- .../config/config.d/perf-comparison-tweaks-config.xml | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 79b351c3721..07e89c6d802 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -22,7 +22,12 @@ if (ENABLE_JEMALLOC) # # By enabling percpu_arena number of arenas limited to number of CPUs and hence # this problem should go away. - set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0") + # + # muzzy_decay_ms -- use MADV_FREE when available on newer Linuxes, to + # avoid spurious latencies and additional work associated with + # MADV_DONTNEED. See + # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. + set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000") else() set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0") endif() diff --git a/docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml b/docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml index 5dcc3c51eca..6f1726ab36b 100644 --- a/docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml +++ b/docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml @@ -20,4 +20,6 @@ 1000000000 + + 10 From 4dcdad2f080cbe15d259b5855ec8f97a5c10a217 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 28 Jun 2020 23:55:45 +0300 Subject: [PATCH 02/24] Fix ORDER BY tuple with COLLATE on const column --- src/Columns/ColumnConst.cpp | 4 +- src/Interpreters/sortBlock.cpp | 3 + ...354_order_by_tuple_collate_const.reference | 66 +++++++++++++++++++ .../01354_order_by_tuple_collate_const.sql | 1 + 4 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01354_order_by_tuple_collate_const.reference create mode 100644 tests/queries/0_stateless/01354_order_by_tuple_collate_const.sql diff --git a/src/Columns/ColumnConst.cpp b/src/Columns/ColumnConst.cpp index 545c0b1b300..2fa1fbce32d 100644 --- a/src/Columns/ColumnConst.cpp +++ b/src/Columns/ColumnConst.cpp @@ -120,7 +120,9 @@ void ColumnConst::getPermutation(bool /*reverse*/, size_t /*limit*/, int /*nan_d res[i] = i; } -void ColumnConst::updatePermutation(bool, size_t, int, Permutation &, EqualRanges &) const {} +void ColumnConst::updatePermutation(bool, size_t, int, Permutation &, EqualRanges &) const +{ +} void ColumnConst::updateWeakHash32(WeakHash32 & hash) const { diff --git a/src/Interpreters/sortBlock.cpp b/src/Interpreters/sortBlock.cpp index 4b8d4f2b859..cb3c36e5356 100644 --- a/src/Interpreters/sortBlock.cpp +++ b/src/Interpreters/sortBlock.cpp @@ -187,6 +187,9 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) if (ranges.empty()) break; + if (column.column_const) + continue; + if (isCollationRequired(column.description)) { const ColumnString & column_string = assert_cast(*column.column); diff --git a/tests/queries/0_stateless/01354_order_by_tuple_collate_const.reference b/tests/queries/0_stateless/01354_order_by_tuple_collate_const.reference new file mode 100644 index 00000000000..0359bf9ccb2 --- /dev/null +++ b/tests/queries/0_stateless/01354_order_by_tuple_collate_const.reference @@ -0,0 +1,66 @@ +0 +0 +1 +1 +10 +10 +2 +2 +3 +3 +4 +4 +5 +5 +6 +6 +7 +7 +8 +8 +9 +9 +0 +1 +10 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +10 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +10 +2 +3 +4 +5 +6 +7 +8 +9 +0 +1 +10 +2 +3 +4 +5 +6 +7 +8 +9 diff --git a/tests/queries/0_stateless/01354_order_by_tuple_collate_const.sql b/tests/queries/0_stateless/01354_order_by_tuple_collate_const.sql new file mode 100644 index 00000000000..d9b596f3ec3 --- /dev/null +++ b/tests/queries/0_stateless/01354_order_by_tuple_collate_const.sql @@ -0,0 +1 @@ +SELECT number FROM numbers(11) ORDER BY arrayJoin(['а', 'я', '\0�', '', 'Я', '']) ASC, toString(number) ASC, 'y' ASC COLLATE 'el'; From 8c0177b21641f7b2fd54d4c36b1986ce551837b5 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 29 Jun 2020 15:01:29 +0300 Subject: [PATCH 03/24] Update contrib/jemalloc-cmake/CMakeLists.txt --- contrib/jemalloc-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 07e89c6d802..13f7ea3326b 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -29,7 +29,7 @@ if (ENABLE_JEMALLOC) # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000") else() - set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0") + set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000") endif() # CACHE variable is empty, to allow changing defaults without necessity # to purge cache From af8d62bbcb1f91b9ef2e452e7b2c49cf2722e39e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 29 Jun 2020 15:48:18 +0300 Subject: [PATCH 04/24] Add TLB misses perf counters. --- src/Common/ProfileEvents.cpp | 8 +++-- src/Common/ThreadProfileEvents.cpp | 54 ++++++++++++++++++++++++---- src/Common/ThreadProfileEvents.h | 10 ++++-- src/Interpreters/ThreadStatusExt.cpp | 4 +++ 4 files changed, 65 insertions(+), 11 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 7a7a6bc6162..5c88b2ee849 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -196,8 +196,12 @@ M(PerfCpuMigrations, "Number of times the process has migrated to a new CPU") \ M(PerfAlignmentFaults, "Number of alignment faults. These happen when unaligned memory accesses happen; the kernel can handle these but it reduces performance. This happens only on some architectures (never on x86).") \ M(PerfEmulationFaults, "Number of emulation faults. The kernel sometimes traps on unimplemented instructions and emulates them for user space. This can negatively impact performance.") \ - M(PerfPageFaultsMinor, "This counts the number of minor page faults. These did not require disk I/O to handle.") \ - M(PerfPageFaultsMajor, "This counts the number of major page faults. These required disk I/O to handle.") \ + M(PerfMinEnabledTime, "For all events, minimum time that an event was enabled. Used to track event multiplexing influence") \ + M(PerfMinEnabledRunningTime, "Running time for event with minimum enabled time. Used to track the amount of event multiplexing") \ + M(PerfDataTLBReferences, "Data TLB references") \ + M(PerfDataTLBMisses, "Data TLB misses") \ + M(PerfInstructionTLBReferences, "Instruction TLB references") \ + M(PerfInstructionTLBMisses, "Instruction TLB misses") \ \ M(CreatedHTTPConnections, "Total amount of created HTTP connections (closed or opened).") \ \ diff --git a/src/Common/ThreadProfileEvents.cpp b/src/Common/ThreadProfileEvents.cpp index 1d65a16ba66..f009096cfec 100644 --- a/src/Common/ThreadProfileEvents.cpp +++ b/src/Common/ThreadProfileEvents.cpp @@ -147,6 +147,19 @@ thread_local PerfEventsCounters current_thread_counters; .settings_name = #LOCAL_NAME \ } +// One event for cache accesses and one for cache misses. +// Type is ACCESS or MISS +#define CACHE_EVENT(PERF_NAME, LOCAL_NAME, TYPE) \ + PerfEventInfo \ + { \ + .event_type = perf_type_id::PERF_TYPE_HW_CACHE, \ + .event_config = (PERF_NAME) \ + | (PERF_COUNT_HW_CACHE_OP_READ << 8) \ + | (PERF_COUNT_HW_CACHE_RESULT_ ## TYPE << 16), \ + .profile_event = ProfileEvents::LOCAL_NAME, \ + .settings_name = #LOCAL_NAME \ + } + // descriptions' source: http://man7.org/linux/man-pages/man2/perf_event_open.2.html static const PerfEventInfo raw_events_info[] = { HARDWARE_EVENT(PERF_COUNT_HW_CPU_CYCLES, PerfCpuCycles), @@ -167,8 +180,16 @@ static const PerfEventInfo raw_events_info[] = { SOFTWARE_EVENT(PERF_COUNT_SW_CPU_MIGRATIONS, PerfCpuMigrations), SOFTWARE_EVENT(PERF_COUNT_SW_ALIGNMENT_FAULTS, PerfAlignmentFaults), SOFTWARE_EVENT(PERF_COUNT_SW_EMULATION_FAULTS, PerfEmulationFaults), - SOFTWARE_EVENT(PERF_COUNT_SW_PAGE_FAULTS_MIN, PerfPageFaultsMinor), - SOFTWARE_EVENT(PERF_COUNT_SW_PAGE_FAULTS_MAJ, PerfPageFaultsMajor) + + // Don't add them -- they are the same as SoftPageFaults and HardPageFaults, + // match well numerically. + // SOFTWARE_EVENT(PERF_COUNT_SW_PAGE_FAULTS_MIN, PerfPageFaultsMinor), + // SOFTWARE_EVENT(PERF_COUNT_SW_PAGE_FAULTS_MAJ, PerfPageFaultsMajor), + + CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfDataTLBReferences, ACCESS), + CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfDataTLBMisses, MISS), + CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfInstructionTLBReferences, ACCESS), + CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfInstructionTLBMisses, MISS), }; static_assert(sizeof(raw_events_info) / sizeof(raw_events_info[0]) == NUMBER_OF_RAW_EVENTS); @@ -455,7 +476,12 @@ void PerfEventsCounters::finalizeProfileEvents(ProfileEvents::Counters & profile } } - // actually process counters' values + // Actually process counters' values. Track the minimal time that a performance + // counter was enabled, and the corresponding running time, to give some idea + // about the amount of counter multiplexing. + UInt64 min_enabled_time = -1; + UInt64 running_time_for_min_enabled_time = 0; + for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i) { int fd = thread_events_descriptors_holder.descriptors[i]; @@ -469,14 +495,30 @@ void PerfEventsCounters::finalizeProfileEvents(ProfileEvents::Counters & profile // Account for counter multiplexing. time_running and time_enabled are // not reset by PERF_EVENT_IOC_RESET, so we don't use it and calculate // deltas from old values. + const auto enabled = current_value.time_enabled - previous_value.time_enabled; + const auto running = current_value.time_running - previous_value.time_running; const UInt64 delta = (current_value.value - previous_value.value) - * (current_value.time_enabled - previous_value.time_enabled) - / std::max(1.f, - float(current_value.time_running - previous_value.time_running)); + * enabled / std::max(1.f, float(running)); + + if (min_enabled_time > enabled) + { + min_enabled_time = enabled; + running_time_for_min_enabled_time = running; + } profile_events.increment(info.profile_event, delta); } + // If we had at least one enabled event, also show multiplexing-related + // statistics. + if (min_enabled_time != UInt64(-1)) + { + profile_events.increment(ProfileEvents::PerfMinEnabledTime, + min_enabled_time); + profile_events.increment(ProfileEvents::PerfMinEnabledRunningTime, + running_time_for_min_enabled_time); + } + // Store current counter values for the next profiling period. memcpy(previous_values, current_values, sizeof(current_values)); } diff --git a/src/Common/ThreadProfileEvents.h b/src/Common/ThreadProfileEvents.h index a4ee0628629..7118e927162 100644 --- a/src/Common/ThreadProfileEvents.h +++ b/src/Common/ThreadProfileEvents.h @@ -53,8 +53,12 @@ namespace ProfileEvents extern const Event PerfCpuMigrations; extern const Event PerfAlignmentFaults; extern const Event PerfEmulationFaults; - extern const Event PerfPageFaultsMinor; - extern const Event PerfPageFaultsMajor; + extern const Event PerfMinEnabledTime; + extern const Event PerfMinEnabledRunningTime; + extern const Event PerfDataTLBReferences; + extern const Event PerfDataTLBMisses; + extern const Event PerfInstructionTLBReferences; + extern const Event PerfInstructionTLBMisses; #endif } @@ -158,7 +162,7 @@ struct PerfEventValue UInt64 time_running = 0; }; -static constexpr size_t NUMBER_OF_RAW_EVENTS = 18; +static constexpr size_t NUMBER_OF_RAW_EVENTS = 20; struct PerfDescriptorsHolder : boost::noncopyable { diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 04265734ce7..e3e695f80f9 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -191,6 +191,10 @@ void ThreadStatus::finalizePerformanceCounters() performance_counters_finalized = true; updatePerformanceCounters(); + // We want to close perf file descriptors if the perf events were enabled for + // one query. What this code does in practice is less clear -- e.g., if I run + // 'select 1 settings metrics_perf_events_enabled = 1', I still get + // query_context->getSettingsRef().metrics_perf_events_enabled == 0 *shrug*. bool close_perf_descriptors = true; if (query_context) close_perf_descriptors = !query_context->getSettingsRef().metrics_perf_events_enabled; From 9b9030caad1c20bc271946a0fb2ee3313223a3ee Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 29 Jun 2020 15:57:36 +0300 Subject: [PATCH 05/24] Update ThreadProfileEvents.cpp --- src/Common/ThreadProfileEvents.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/ThreadProfileEvents.cpp b/src/Common/ThreadProfileEvents.cpp index f009096cfec..0e5d5c78a6f 100644 --- a/src/Common/ThreadProfileEvents.cpp +++ b/src/Common/ThreadProfileEvents.cpp @@ -188,8 +188,8 @@ static const PerfEventInfo raw_events_info[] = { CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfDataTLBReferences, ACCESS), CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfDataTLBMisses, MISS), - CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfInstructionTLBReferences, ACCESS), - CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfInstructionTLBMisses, MISS), + CACHE_EVENT(PERF_COUNT_HW_CACHE_ITLB, PerfInstructionTLBReferences, ACCESS), + CACHE_EVENT(PERF_COUNT_HW_CACHE_ITLB, PerfInstructionTLBMisses, MISS), }; static_assert(sizeof(raw_events_info) / sizeof(raw_events_info[0]) == NUMBER_OF_RAW_EVENTS); From eb27814fbe9d6e3151fcd170d2d17a168176b3dc Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 28 Jun 2020 21:39:44 +0300 Subject: [PATCH 06/24] Fix access rights: cannot grant INTROSPECTION when allow_introspection_functions=0. --- src/Access/ContextAccess.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index e7bd0f8287d..62aebfd4367 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -431,7 +431,7 @@ boost::shared_ptr ContextAccess::calculateResultAccess(bool if (!allow_ddl_) merged_access->revoke(table_and_dictionary_ddl); - if (!allow_introspection_ && !grant_option) + if (!allow_introspection_) merged_access->revoke(AccessType::INTROSPECTION); /// Anyone has access to the "system" database. From 877c10f5fabc6a51c4b9068d4e4b049e30a50c19 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 29 Jun 2020 17:13:16 +0300 Subject: [PATCH 07/24] Update ThreadProfileEvents.cpp --- src/Common/ThreadProfileEvents.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ThreadProfileEvents.cpp b/src/Common/ThreadProfileEvents.cpp index 0e5d5c78a6f..c63050792c2 100644 --- a/src/Common/ThreadProfileEvents.cpp +++ b/src/Common/ThreadProfileEvents.cpp @@ -180,7 +180,7 @@ static const PerfEventInfo raw_events_info[] = { SOFTWARE_EVENT(PERF_COUNT_SW_CPU_MIGRATIONS, PerfCpuMigrations), SOFTWARE_EVENT(PERF_COUNT_SW_ALIGNMENT_FAULTS, PerfAlignmentFaults), SOFTWARE_EVENT(PERF_COUNT_SW_EMULATION_FAULTS, PerfEmulationFaults), - + // Don't add them -- they are the same as SoftPageFaults and HardPageFaults, // match well numerically. // SOFTWARE_EVENT(PERF_COUNT_SW_PAGE_FAULTS_MIN, PerfPageFaultsMinor), From 84fa7fa3d89f8420b3cf7044165838d8f04fcf45 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 30 Jun 2020 01:51:49 +0300 Subject: [PATCH 08/24] Remove test that is not supported by "Arcadia" build system --- tests/queries/0_stateless/arcadia_skip_list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 2b61d384b00..3995f905332 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -125,3 +125,4 @@ 01326_build_id 01053_ssd_dictionary 01280_ssd_complex_key_dictionary +01354_order_by_tuple_collate_const From 15cddc3c6e659cec18f4b16f7fc4cd3b3b36a1cb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 30 Jun 2020 02:17:20 +0300 Subject: [PATCH 09/24] Make topK return Enum for Enum types --- src/AggregateFunctions/AggregateFunctionTopK.cpp | 3 ++- src/AggregateFunctions/AggregateFunctionTopK.h | 2 +- tests/queries/0_stateless/01353_topk_enum.reference | 1 + tests/queries/0_stateless/01353_topk_enum.sql | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01353_topk_enum.reference create mode 100644 tests/queries/0_stateless/01353_topk_enum.sql diff --git a/src/AggregateFunctions/AggregateFunctionTopK.cpp b/src/AggregateFunctions/AggregateFunctionTopK.cpp index 7f2da260c2d..344ab340d62 100644 --- a/src/AggregateFunctions/AggregateFunctionTopK.cpp +++ b/src/AggregateFunctions/AggregateFunctionTopK.cpp @@ -100,7 +100,8 @@ AggregateFunctionPtr createAggregateFunctionTopK(const std::string & name, const threshold = k; } - AggregateFunctionPtr res(createWithNumericType(*argument_types[0], threshold, load_factor, argument_types, params)); + AggregateFunctionPtr res(createWithNumericType( + *argument_types[0], threshold, load_factor, argument_types, params)); if (!res) res = AggregateFunctionPtr(createWithExtraTypes(argument_types[0], threshold, load_factor, params)); diff --git a/src/AggregateFunctions/AggregateFunctionTopK.h b/src/AggregateFunctions/AggregateFunctionTopK.h index 68317d0bdf0..f77fc482685 100644 --- a/src/AggregateFunctions/AggregateFunctionTopK.h +++ b/src/AggregateFunctions/AggregateFunctionTopK.h @@ -47,7 +47,7 @@ public: DataTypePtr getReturnType() const override { - return std::make_shared(std::make_shared>()); + return std::make_shared(this->argument_types[0]); } void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override diff --git a/tests/queries/0_stateless/01353_topk_enum.reference b/tests/queries/0_stateless/01353_topk_enum.reference new file mode 100644 index 00000000000..d650850c434 --- /dev/null +++ b/tests/queries/0_stateless/01353_topk_enum.reference @@ -0,0 +1 @@ +['test','world','hello',''] diff --git a/tests/queries/0_stateless/01353_topk_enum.sql b/tests/queries/0_stateless/01353_topk_enum.sql new file mode 100644 index 00000000000..ba048401b23 --- /dev/null +++ b/tests/queries/0_stateless/01353_topk_enum.sql @@ -0,0 +1 @@ +WITH CAST(round(sqrt(number)) % 4 AS Enum('' = 0, 'hello' = 1, 'world' = 2, 'test' = 3)) AS x SELECT topK(10)(x) FROM numbers(1000); From 987e64acfe789689b8710add5d04371b8f21b604 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 30 Jun 2020 02:28:25 +0300 Subject: [PATCH 10/24] parse metadata in parallel when loading tables --- src/Databases/DatabaseOnDisk.cpp | 21 ++++++- src/Databases/DatabaseOrdinary.cpp | 4 +- .../01193_metadata_loading.reference | 5 ++ .../0_stateless/01193_metadata_loading.sh | 59 +++++++++++++++++++ 4 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/01193_metadata_loading.reference create mode 100755 tests/queries/0_stateless/01193_metadata_loading.sh diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 0a16b6eacff..2ad7af9c703 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -389,6 +389,9 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati } }; + /// Metadata files to load: name and flag for .tmp_drop files + std::set> metadata_files; + Poco::DirectoryIterator dir_end; for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it) { @@ -404,7 +407,7 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati if (endsWith(dir_it.name(), tmp_drop_ext)) { /// There are files that we tried to delete previously - process_tmp_drop_metadata_file(dir_it.name()); + metadata_files.emplace(dir_it.name(), false); } else if (endsWith(dir_it.name(), ".sql.tmp")) { @@ -415,12 +418,26 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const Iterati else if (endsWith(dir_it.name(), ".sql")) { /// The required files have names like `table_name.sql` - process_metadata_file(dir_it.name()); + metadata_files.emplace(dir_it.name(), true); } else throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + getMetadataPath(), ErrorCodes::INCORRECT_FILE_NAME); } + + /// Read and parse metadata in parallel + ThreadPool pool(SettingMaxThreads().getAutoValue()); + for (const auto & file : metadata_files) + { + pool.scheduleOrThrowOnError([&]() + { + if (file.second) + process_metadata_file(file.first); + else + process_tmp_drop_metadata_file(file.first); + }); + } + pool.wait(); } ASTPtr DatabaseOnDisk::parseQueryFromMetadata(Poco::Logger * loger, const Context & context, const String & metadata_file_path, bool throw_on_error /*= true*/, bool remove_empty /*= false*/) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index eec58ed9b33..277588b59f0 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -121,11 +121,12 @@ void DatabaseOrdinary::loadStoredObjects( * which does not correspond to order tables creation and does not correspond to order of their location on disk. */ using FileNames = std::map; + std::mutex file_names_mutex; FileNames file_names; size_t total_dictionaries = 0; - auto process_metadata = [&context, &file_names, &total_dictionaries, this](const String & file_name) + auto process_metadata = [&context, &file_names, &total_dictionaries, &file_names_mutex, this](const String & file_name) { String full_path = getMetadataPath() + file_name; try @@ -134,6 +135,7 @@ void DatabaseOrdinary::loadStoredObjects( if (ast) { auto * create_query = ast->as(); + std::lock_guard lock{file_names_mutex}; file_names[file_name] = ast; total_dictionaries += create_query->is_dictionary; } diff --git a/tests/queries/0_stateless/01193_metadata_loading.reference b/tests/queries/0_stateless/01193_metadata_loading.reference new file mode 100644 index 00000000000..8ff246325ac --- /dev/null +++ b/tests/queries/0_stateless/01193_metadata_loading.reference @@ -0,0 +1,5 @@ +10000 0 2020-06-25 hello [1,2] [3,4] +10000 1 2020-06-26 word [10,20] [30,40] +ok +8000 0 2020-06-25 hello [1,2] [3,4] +8000 1 2020-06-26 word [10,20] [30,40] diff --git a/tests/queries/0_stateless/01193_metadata_loading.sh b/tests/queries/0_stateless/01193_metadata_loading.sh new file mode 100755 index 00000000000..de74b3ec1af --- /dev/null +++ b/tests/queries/0_stateless/01193_metadata_loading.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +# it is the worst way of making performance test, nevertheless it can detect significant slowdown and some other issues, that usually found by stress test + +db="test_01193_$RANDOM" + +declare -A engines +engines[0]="Memory" +engines[1]="File(CSV)" +engines[2]="Log" +engines[3]="StripeLog" +engines[4]="MergeTree ORDER BY i" + +tables=1000 +threads=10 +count_multiplier=1 +max_time_ms=1000 + +debug_or_sanitizer_build=`$CLICKHOUSE_CLIENT -q "WITH ((SELECT value FROM system.build_options WHERE name='BUILD_TYPE') AS build, (SELECT value FROM system.build_options WHERE name='CXX_FLAGS') as flags) SELECT build='Debug' OR flags LIKE '%fsanitize%'"` + +if [[ debug_or_sanitizer_build -eq 1 ]]; then tables=100; count_multiplier=10; max_time_ms=1500; fi + +create_tables() { + for i in $(seq 1 $tables); do + engine=${engines[$((i % ${#engines[@]}))]} + $CLICKHOUSE_CLIENT -q "CREATE TABLE $db.table_$1_$i (i UInt64, d Date, s String, n Nested(i UInt8, f Float32)) ENGINE=$engine" + $CLICKHOUSE_CLIENT -q "INSERT INTO $db.table_$1_$i VALUES (0, '2020-06-25', 'hello', [1, 2], [3, 4]), (1, '2020-06-26', 'word', [10, 20], [30, 40])" + done +} + +$CLICKHOUSE_CLIENT -q "CREATE DATABASE $db" + +for i in $(seq 1 $threads); do + create_tables $i & +done +wait + +$CLICKHOUSE_CLIENT -q "CREATE TABLE $db.table_merge (i UInt64, d Date, s String, n Nested(i UInt8, f Float32)) ENGINE=Merge('$db', '^table_')" +#FIXME the following query leads to segfault +#$CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM $db.table_merge GROUP BY i, d, s, n.i, n.f ORDER BY i" +$CLICKHOUSE_CLIENT -q "SELECT 10000, i, d, s, n.i, n.f FROM $db.table_1_1 GROUP BY i, d, s, n.i, n.f ORDER BY i" + +db_engine=`$CLICKHOUSE_CLIENT -q "SELECT engine FROM system.databases WHERE name='$db'"` + +$CLICKHOUSE_CLIENT -q "DETACH DATABASE $db" + +# get real time, grep seconds, remove point, remove leading zeros +elapsed_ms=`{ time $CLICKHOUSE_CLIENT -q "ATTACH DATABASE $db ENGINE=$db_engine"; } 2>&1 | grep real | grep -Po "0m\K[0-9\.]*" | tr -d '.' | sed "s/^0*//"` +$CLICKHOUSE_CLIENT -q "SELECT '01193_metadata_loading', $elapsed_ms FORMAT Null" # it will be printed to server log + +if [[ $elapsed_ms -le $max_time_ms ]]; then echo ok; fi + +#$CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM $db.table_merge GROUP BY i, d, s, n.i, n.f ORDER BY i" +$CLICKHOUSE_CLIENT -q "SELECT 8000, i, d, s, n.i, n.f FROM $db.table_1_1 GROUP BY i, d, s, n.i, n.f ORDER BY i" + +$CLICKHOUSE_CLIENT -q "DROP DATABASE $db" From 1d838b7b3ac6b80ca7ce47f43202ffbd710b1de8 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 30 Jun 2020 09:56:11 +0300 Subject: [PATCH 11/24] Update ThreadProfileEvents.cpp --- src/Common/ThreadProfileEvents.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Common/ThreadProfileEvents.cpp b/src/Common/ThreadProfileEvents.cpp index c63050792c2..0a0b89aa4bf 100644 --- a/src/Common/ThreadProfileEvents.cpp +++ b/src/Common/ThreadProfileEvents.cpp @@ -188,6 +188,9 @@ static const PerfEventInfo raw_events_info[] = { CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfDataTLBReferences, ACCESS), CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfDataTLBMisses, MISS), + + // Apparently it doesn't make sense to treat these values as relative: + // https://stackoverflow.com/questions/49933319/how-to-interpret-perf-itlb-loads-itlb-load-misses CACHE_EVENT(PERF_COUNT_HW_CACHE_ITLB, PerfInstructionTLBReferences, ACCESS), CACHE_EVENT(PERF_COUNT_HW_CACHE_ITLB, PerfInstructionTLBMisses, MISS), }; From aa8461e99667bd26950d52cc71c672a34b10ff7d Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 30 Jun 2020 10:47:19 +0300 Subject: [PATCH 12/24] Update ThreadProfileEvents.cpp --- src/Common/ThreadProfileEvents.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ThreadProfileEvents.cpp b/src/Common/ThreadProfileEvents.cpp index 0a0b89aa4bf..04c29841e23 100644 --- a/src/Common/ThreadProfileEvents.cpp +++ b/src/Common/ThreadProfileEvents.cpp @@ -188,7 +188,7 @@ static const PerfEventInfo raw_events_info[] = { CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfDataTLBReferences, ACCESS), CACHE_EVENT(PERF_COUNT_HW_CACHE_DTLB, PerfDataTLBMisses, MISS), - + // Apparently it doesn't make sense to treat these values as relative: // https://stackoverflow.com/questions/49933319/how-to-interpret-perf-itlb-loads-itlb-load-misses CACHE_EVENT(PERF_COUNT_HW_CACHE_ITLB, PerfInstructionTLBReferences, ACCESS), From ace9c99e14668a00147a1bb51f9dd888baa0dfcd Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 28 Jun 2020 21:41:41 +0300 Subject: [PATCH 13/24] Fix test_allow_introspection. --- .../configs/{ => config.d}/remote_servers.xml | 0 .../users.d/allow_introspection_functions.xml | 7 +++++++ .../test_settings_constraints_distributed/test.py | 6 +++--- tests/integration/test_settings_profile/test.py | 15 +++++++++++---- 4 files changed, 21 insertions(+), 7 deletions(-) rename tests/integration/test_settings_constraints_distributed/configs/{ => config.d}/remote_servers.xml (100%) create mode 100644 tests/integration/test_settings_constraints_distributed/configs/users.d/allow_introspection_functions.xml diff --git a/tests/integration/test_settings_constraints_distributed/configs/remote_servers.xml b/tests/integration/test_settings_constraints_distributed/configs/config.d/remote_servers.xml similarity index 100% rename from tests/integration/test_settings_constraints_distributed/configs/remote_servers.xml rename to tests/integration/test_settings_constraints_distributed/configs/config.d/remote_servers.xml diff --git a/tests/integration/test_settings_constraints_distributed/configs/users.d/allow_introspection_functions.xml b/tests/integration/test_settings_constraints_distributed/configs/users.d/allow_introspection_functions.xml new file mode 100644 index 00000000000..ccfdf6a63f6 --- /dev/null +++ b/tests/integration/test_settings_constraints_distributed/configs/users.d/allow_introspection_functions.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/integration/test_settings_constraints_distributed/test.py b/tests/integration/test_settings_constraints_distributed/test.py index 86456f8a099..7f0f8868bcf 100644 --- a/tests/integration/test_settings_constraints_distributed/test.py +++ b/tests/integration/test_settings_constraints_distributed/test.py @@ -8,9 +8,9 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1') -node2 = cluster.add_instance('node2') -distributed = cluster.add_instance('distributed', main_configs=["configs/remote_servers.xml"], stay_alive=True) +node1 = cluster.add_instance('node1', config_dir="configs") +node2 = cluster.add_instance('node2', config_dir="configs") +distributed = cluster.add_instance('distributed', config_dir="configs", stay_alive=True) @pytest.fixture(scope="module", autouse=True) diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 752aa2da75d..21fdac9da7a 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -177,11 +177,18 @@ def test_allow_ddl(): def test_allow_introspection(): + assert "Introspection functions are disabled" in instance.query_and_get_error("SELECT demangle('a')") assert "Not enough privileges" in instance.query_and_get_error("SELECT demangle('a')", user="robin") - - instance.query("GRANT ALL ON *.* TO robin") - assert "Introspection functions are disabled" in instance.query_and_get_error("SELECT demangle('a')", user="robin") + assert "Not enough privileges" in instance.query_and_get_error("SELECT demangle('a')", user="robin", settings={"allow_introspection_functions":1}) + assert "Introspection functions are disabled" in instance.query_and_get_error("GRANT demangle ON *.* TO robin") + assert "Not enough privileges" in instance.query_and_get_error("GRANT demangle ON *.* TO robin", user="robin") + assert "Not enough privileges" in instance.query_and_get_error("GRANT demangle ON *.* TO robin", user="robin", settings={"allow_introspection_functions":1}) + + assert instance.query("SELECT demangle('a')", settings={"allow_introspection_functions":1}) == "signed char\n" + instance.query("GRANT demangle ON *.* TO robin", settings={"allow_introspection_functions":1}) + + assert "Introspection functions are disabled" in instance.query_and_get_error("SELECT demangle('a')", user="robin") instance.query("ALTER USER robin SETTINGS allow_introspection_functions=1") assert instance.query("SELECT demangle('a')", user="robin") == "signed char\n" @@ -194,5 +201,5 @@ def test_allow_introspection(): instance.query("DROP SETTINGS PROFILE xyz") assert "Introspection functions are disabled" in instance.query_and_get_error("SELECT demangle('a')", user="robin") - instance.query("REVOKE ALL ON *.* FROM robin") + instance.query("REVOKE demangle ON *.* FROM robin", settings={"allow_introspection_functions":1}) assert "Not enough privileges" in instance.query_and_get_error("SELECT demangle('a')", user="robin") From 29178e26daa4cb88690058145ae3a5ab5d724c7f Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Tue, 30 Jun 2020 14:13:43 +0300 Subject: [PATCH 14/24] fix low card types in merge join (#12035) --- src/Interpreters/MergeJoin.cpp | 5 +- src/Interpreters/join_common.cpp | 10 +++ src/Interpreters/join_common.h | 1 + ...00800_low_cardinality_merge_join.reference | 41 ++++++++++ .../00800_low_cardinality_merge_join.sql | 30 ++++++++ ...01353_low_cardinality_join_types.reference | 36 +++++++++ .../01353_low_cardinality_join_types.sql | 75 +++++++++++++++++++ 7 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/00800_low_cardinality_merge_join.reference create mode 100644 tests/queries/0_stateless/00800_low_cardinality_merge_join.sql create mode 100644 tests/queries/0_stateless/01353_low_cardinality_join_types.reference create mode 100644 tests/queries/0_stateless/01353_low_cardinality_join_types.sql diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index bb054169e71..1478c36dd23 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -396,7 +396,6 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & right if (required_right_keys.count(column.name)) right_columns_to_add.insert(ColumnWithTypeAndName{nullptr, column.type, column.name}); - JoinCommon::removeLowCardinalityInplace(right_columns_to_add); JoinCommon::createMissedColumns(right_columns_to_add); if (nullable_right_side) @@ -513,7 +512,7 @@ bool MergeJoin::saveRightBlock(Block && block) bool MergeJoin::addJoinedBlock(const Block & src_block, bool) { Block block = materializeBlock(src_block); - JoinCommon::removeLowCardinalityInplace(block); + JoinCommon::removeLowCardinalityInplace(block, table_join->keyNamesRight()); sortBlock(block, right_sort_description); return saveRightBlock(std::move(block)); @@ -525,7 +524,7 @@ void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) { JoinCommon::checkTypesOfKeys(block, table_join->keyNamesLeft(), right_table_keys, table_join->keyNamesRight()); materializeBlockInplace(block); - JoinCommon::removeLowCardinalityInplace(block); + JoinCommon::removeLowCardinalityInplace(block, table_join->keyNamesLeft()); sortBlock(block, left_sort_description); } diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index 6dd3a202d4d..a17d3b43e69 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -104,6 +104,16 @@ void removeLowCardinalityInplace(Block & block) } } +void removeLowCardinalityInplace(Block & block, const Names & names) +{ + for (const String & column_name : names) + { + auto & col = block.getByName(column_name); + col.column = recursiveRemoveLowCardinality(col.column); + col.type = recursiveRemoveLowCardinality(col.type); + } +} + void splitAdditionalColumns(const Block & sample_block, const Names & key_names, Block & block_keys, Block & block_others) { block_others = materializeBlock(sample_block); diff --git a/src/Interpreters/join_common.h b/src/Interpreters/join_common.h index 47fa082e700..81eb0dfa688 100644 --- a/src/Interpreters/join_common.h +++ b/src/Interpreters/join_common.h @@ -20,6 +20,7 @@ Columns materializeColumns(const Block & block, const Names & names); ColumnRawPtrs materializeColumnsInplace(Block & block, const Names & names); ColumnRawPtrs getRawPointers(const Columns & columns); void removeLowCardinalityInplace(Block & block); +void removeLowCardinalityInplace(Block & block, const Names & names); /// Split key and other columns by keys name list void splitAdditionalColumns(const Block & sample_block, const Names & key_names, Block & block_keys, Block & block_others); diff --git a/tests/queries/0_stateless/00800_low_cardinality_merge_join.reference b/tests/queries/0_stateless/00800_low_cardinality_merge_join.reference new file mode 100644 index 00000000000..8e032c0a542 --- /dev/null +++ b/tests/queries/0_stateless/00800_low_cardinality_merge_join.reference @@ -0,0 +1,41 @@ +0 +0 +0 +0 +0 +0 +0 +0 +0 +- +0 0 +0 0 +0 0 +0 0 +0 0 +0 0 +0 0 +0 0 +0 0 +- +0 1 +1 2 +2 0 +0 1 +1 2 +2 0 +0 1 +1 2 +2 0 +0 1 +1 2 +2 0 +0 1 +1 2 +2 0 +0 1 +1 2 +2 \N +0 1 +1 2 +2 \N diff --git a/tests/queries/0_stateless/00800_low_cardinality_merge_join.sql b/tests/queries/0_stateless/00800_low_cardinality_merge_join.sql new file mode 100644 index 00000000000..1181ee453a6 --- /dev/null +++ b/tests/queries/0_stateless/00800_low_cardinality_merge_join.sql @@ -0,0 +1,30 @@ +set join_algorithm = 'partial_merge'; + +select * from (select dummy as val from system.one) s1 any left join (select dummy as val from system.one) s2 using val; +select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select dummy as val from system.one) s2 using val; +select * from (select dummy as val from system.one) s1 any left join (select toLowCardinality(dummy) as val from system.one) s2 using val; +select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select toLowCardinality(dummy) as val from system.one) s2 using val; +select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select dummy as val from system.one) s2 using val; +select * from (select dummy as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as val from system.one) s2 using val; +select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select toLowCardinality(dummy) as val from system.one) s2 using val; +select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as val from system.one) s2 using val; +select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as val from system.one) s2 using val; +select '-'; +select * from (select dummy as val from system.one) s1 any left join (select dummy as val from system.one) s2 on val + 0 = val * 1; -- { serverError 352 } +select * from (select dummy as val from system.one) s1 any left join (select dummy as rval from system.one) s2 on val + 0 = rval * 1; +select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select dummy as rval from system.one) s2 on val + 0 = rval * 1; +select * from (select dummy as val from system.one) s1 any left join (select toLowCardinality(dummy) as rval from system.one) s2 on val + 0 = rval * 1; +select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select toLowCardinality(dummy) as rval from system.one) s2 on val + 0 = rval * 1; +select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select dummy as rval from system.one) s2 on val + 0 = rval * 1; +select * from (select dummy as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as rval from system.one) s2 on val + 0 = rval * 1; +select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select toLowCardinality(dummy) as rval from system.one) s2 on val + 0 = rval * 1; +select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as rval from system.one) s2 on val + 0 = rval * 1; +select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as rval from system.one) s2 on val + 0 = rval * 1; +select '-'; +select * from (select number as l from system.numbers limit 3) s1 any left join (select number as r from system.numbers limit 3) s2 on l + 1 = r * 1; +select * from (select toLowCardinality(number) as l from system.numbers limit 3) s1 any left join (select number as r from system.numbers limit 3) s2 on l + 1 = r * 1; +select * from (select number as l from system.numbers limit 3) s1 any left join (select toLowCardinality(number) as r from system.numbers limit 3) s2 on l + 1 = r * 1; +select * from (select toLowCardinality(number) as l from system.numbers limit 3) s1 any left join (select toLowCardinality(number) as r from system.numbers limit 3) s2 on l + 1 = r * 1; +select * from (select toLowCardinality(toNullable(number)) as l from system.numbers limit 3) s1 any left join (select toLowCardinality(number) as r from system.numbers limit 3) s2 on l + 1 = r * 1; +select * from (select toLowCardinality(number) as l from system.numbers limit 3) s1 any left join (select toLowCardinality(toNullable(number)) as r from system.numbers limit 3) s2 on l + 1 = r * 1; +select * from (select toLowCardinality(toNullable(number)) as l from system.numbers limit 3) s1 any left join (select toLowCardinality(toNullable(number)) as r from system.numbers limit 3) s2 on l + 1 = r * 1; diff --git a/tests/queries/0_stateless/01353_low_cardinality_join_types.reference b/tests/queries/0_stateless/01353_low_cardinality_join_types.reference new file mode 100644 index 00000000000..85d3f3d598b --- /dev/null +++ b/tests/queries/0_stateless/01353_low_cardinality_join_types.reference @@ -0,0 +1,36 @@ +- +LowCardinality(UInt64) UInt64 String LowCardinality(String) +- +UInt64 LowCardinality(UInt64) LowCardinality(String) String +- +LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String) +- +LowCardinality(UInt64) UInt64 String LowCardinality(String) +LowCardinality(UInt64) UInt64 String LowCardinality(String) +LowCardinality(UInt64) UInt64 String LowCardinality(String) +- +UInt64 LowCardinality(UInt64) LowCardinality(String) String +UInt64 LowCardinality(UInt64) LowCardinality(String) String +UInt64 LowCardinality(UInt64) LowCardinality(String) String +- +LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String) +LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String) +LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String) +- +LowCardinality(UInt64) UInt64 String LowCardinality(String) +- +UInt64 LowCardinality(UInt64) LowCardinality(String) String +- +LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String) +- +LowCardinality(UInt64) UInt64 String LowCardinality(String) +LowCardinality(UInt64) UInt64 String LowCardinality(String) +LowCardinality(UInt64) UInt64 String LowCardinality(String) +- +UInt64 LowCardinality(UInt64) LowCardinality(String) String +UInt64 LowCardinality(UInt64) LowCardinality(String) String +UInt64 LowCardinality(UInt64) LowCardinality(String) String +- +LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String) +LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String) +LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String) diff --git a/tests/queries/0_stateless/01353_low_cardinality_join_types.sql b/tests/queries/0_stateless/01353_low_cardinality_join_types.sql new file mode 100644 index 00000000000..91ebe97fa48 --- /dev/null +++ b/tests/queries/0_stateless/01353_low_cardinality_join_types.sql @@ -0,0 +1,75 @@ +set join_algorithm = 'hash'; + +select '-'; +select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s)) +from (select toLowCardinality(number) k, toString(number) s from numbers(2)) as js1 +join (select number+1 k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2 +using k order by js1.k, js2.k; + +select '-'; +select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s)) +from (select number k, toLowCardinality(toString(number)) s from numbers(2)) as js1 +join (select toLowCardinality(number+1) k, toString(number+1) s from numbers(2)) as js2 +using k order by js1.k, js2.k; + +select '-'; +select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s)) +from (select toLowCardinality(number) k, toLowCardinality(toString(number)) s from numbers(2)) as js1 +join (select toLowCardinality(number+1) k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2 +using k order by js1.k, js2.k; + +select '-'; +select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s)) +from (select toLowCardinality(number) k, toString(number) s from numbers(2)) as js1 +full join (select number+1 k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2 +using k order by js1.k, js2.k; + +select '-'; +select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s)) +from (select number k, toLowCardinality(toString(number)) s from numbers(2)) as js1 +full join (select toLowCardinality(number+1) k, toString(number+1) s from numbers(2)) as js2 +using k order by js1.k, js2.k; + +select '-'; +select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s)) +from (select toLowCardinality(number) k, toLowCardinality(toString(number)) s from numbers(2)) as js1 +full join (select toLowCardinality(number+1) k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2 +using k order by js1.k, js2.k; + +set join_algorithm = 'prefer_partial_merge'; + +select '-'; +select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s)) +from (select toLowCardinality(number) k, toString(number) s from numbers(2)) as js1 +join (select number+1 k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2 +using k order by js1.k, js2.k; + +select '-'; +select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s)) +from (select number k, toLowCardinality(toString(number)) s from numbers(2)) as js1 +join (select toLowCardinality(number+1) k, toString(number+1) s from numbers(2)) as js2 +using k order by js1.k, js2.k; + +select '-'; +select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s)) +from (select toLowCardinality(number) k, toLowCardinality(toString(number)) s from numbers(2)) as js1 +join (select toLowCardinality(number+1) k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2 +using k order by js1.k, js2.k; + +select '-'; +select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s)) +from (select toLowCardinality(number) k, toString(number) s from numbers(2)) as js1 +full join (select number+1 k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2 +using k order by js1.k, js2.k; + +select '-'; +select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s)) +from (select number k, toLowCardinality(toString(number)) s from numbers(2)) as js1 +full join (select toLowCardinality(number+1) k, toString(number+1) s from numbers(2)) as js2 +using k order by js1.k, js2.k; + +select '-'; +select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s)) +from (select toLowCardinality(number) k, toLowCardinality(toString(number)) s from numbers(2)) as js1 +full join (select toLowCardinality(number+1) k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2 +using k order by js1.k, js2.k; From 8b2a5d81df73cc32453fb7a7d5c53876dfbc20b9 Mon Sep 17 00:00:00 2001 From: Alexander Kazakov Date: Tue, 30 Jun 2020 14:27:40 +0300 Subject: [PATCH 15/24] Update contrib/poco to ClickHouse-Extras/poco #22 (#12037) * Updated ClickHouse-Extras/poco #22 --- .gitmodules | 2 +- contrib/poco | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 2fed57a519d..c767d4f41fe 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "contrib/poco"] path = contrib/poco - url = https://github.com/ClickHouse-Extras/poco + url = https://github.com/ClickHouse-Extras/poco.git branch = clickhouse [submodule "contrib/zstd"] path = contrib/zstd diff --git a/contrib/poco b/contrib/poco index be2ab90ba5d..74c93443342 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit be2ab90ba5dccd46919a116e3fe4fa77bb85063b +Subproject commit 74c93443342f6028fa6402057684733b316aa737 From f4869eca41d0c737fd507216a20230cb3d208850 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 30 Jun 2020 15:14:18 +0300 Subject: [PATCH 16/24] Update cpu_synthetic.xml --- tests/performance/cpu_synthetic.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/cpu_synthetic.xml b/tests/performance/cpu_synthetic.xml index 2888f7bbbd6..e08e06f9833 100644 --- a/tests/performance/cpu_synthetic.xml +++ b/tests/performance/cpu_synthetic.xml @@ -21,7 +21,7 @@ PageCharset тоже почти всегда непуст, но его сред SELECT count() FROM hits_10m_single WHERE NOT ignore(sipHash64(SearchPhrase)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(sipHash64(SearchPhrase)) -SELECT count() FROM hits_10m_single WHERE NOT ignore(MD5(SearchPhrase)) SETTINGS max_threads = 1 + SELECT count() FROM hits_100m_single WHERE NOT ignore(MD5(SearchPhrase)) From c39eb8f71b6bc7d277bc92c046b4899dfeda3c41 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 21 Jun 2020 01:44:52 +0300 Subject: [PATCH 17/24] Fix partial revokes (complex cases). --- src/Access/AccessControlManager.cpp | 41 +- src/Access/AccessControlManager.h | 3 + src/Access/AccessRights.cpp | 696 ++++++++++++------ src/Access/AccessRights.h | 91 ++- src/Access/AccessRightsElement.cpp | 288 +++----- src/Access/AccessRightsElement.h | 85 ++- src/Access/ContextAccess.cpp | 657 ++++++++--------- src/Access/ContextAccess.h | 132 ++-- src/Access/EnabledRolesInfo.cpp | 3 +- src/Access/EnabledRolesInfo.h | 1 - src/Access/GrantedAccess.cpp | 22 - src/Access/GrantedAccess.h | 55 -- src/Access/Role.h | 4 +- src/Access/RoleCache.cpp | 3 +- src/Access/User.h | 4 +- src/Access/ya.make | 1 - src/Interpreters/DDLWorker.cpp | 37 +- src/Interpreters/DDLWorker.h | 3 +- src/Interpreters/InterpreterGrantQuery.cpp | 31 +- .../InterpreterKillQueryQuery.cpp | 22 +- .../InterpreterShowGrantsQuery.cpp | 57 +- src/Interpreters/InterpreterSystemQuery.cpp | 10 +- src/Interpreters/tests/users.cpp | 2 +- src/Parsers/ASTGrantQuery.cpp | 6 + src/Parsers/ASTGrantQuery.h | 7 +- src/Storages/System/StorageSystemGrants.cpp | 26 +- .../test_create_user_and_login/test.py | 1 - tests/integration/test_role/test.py | 1 - .../01073_grant_and_revoke.reference | 4 +- .../01074_partial_revokes.reference | 59 ++ .../0_stateless/01074_partial_revokes.sql | 98 +++ 31 files changed, 1378 insertions(+), 1072 deletions(-) delete mode 100644 src/Access/GrantedAccess.cpp delete mode 100644 src/Access/GrantedAccess.h diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index 1c1215a0e28..94a45e3e1c1 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -40,27 +40,8 @@ class AccessControlManager::ContextAccessCache public: explicit ContextAccessCache(const AccessControlManager & manager_) : manager(manager_) {} - std::shared_ptr getContextAccess( - const UUID & user_id, - const boost::container::flat_set & current_roles, - bool use_default_roles, - const Settings & settings, - const String & current_database, - const ClientInfo & client_info) + std::shared_ptr getContextAccess(const ContextAccessParams & params) { - ContextAccess::Params params; - params.user_id = user_id; - params.current_roles = current_roles; - params.use_default_roles = use_default_roles; - params.current_database = current_database; - params.readonly = settings.readonly; - params.allow_ddl = settings.allow_ddl; - params.allow_introspection = settings.allow_introspection_functions; - params.interface = client_info.interface; - params.http_method = client_info.http_method; - params.address = client_info.current_address.host(); - params.quota_key = client_info.quota_key; - std::lock_guard lock{mutex}; auto x = cache.get(params); if (x) @@ -119,7 +100,25 @@ std::shared_ptr AccessControlManager::getContextAccess( const String & current_database, const ClientInfo & client_info) const { - return context_access_cache->getContextAccess(user_id, current_roles, use_default_roles, settings, current_database, client_info); + ContextAccessParams params; + params.user_id = user_id; + params.current_roles = current_roles; + params.use_default_roles = use_default_roles; + params.current_database = current_database; + params.readonly = settings.readonly; + params.allow_ddl = settings.allow_ddl; + params.allow_introspection = settings.allow_introspection_functions; + params.interface = client_info.interface; + params.http_method = client_info.http_method; + params.address = client_info.current_address.host(); + params.quota_key = client_info.quota_key; + return getContextAccess(params); +} + + +std::shared_ptr AccessControlManager::getContextAccess(const ContextAccessParams & params) const +{ + return context_access_cache->getContextAccess(params); } diff --git a/src/Access/AccessControlManager.h b/src/Access/AccessControlManager.h index 6bcf8d7c504..d244ecd07d2 100644 --- a/src/Access/AccessControlManager.h +++ b/src/Access/AccessControlManager.h @@ -21,6 +21,7 @@ namespace Poco namespace DB { class ContextAccess; +struct ContextAccessParams; struct User; using UserPtr = std::shared_ptr; class EnabledRoles; @@ -58,6 +59,8 @@ public: const String & current_database, const ClientInfo & client_info) const; + std::shared_ptr getContextAccess(const ContextAccessParams & params) const; + std::shared_ptr getEnabledRoles( const boost::container::flat_set & current_roles, const boost::container::flat_set & current_roles_with_admin_option) const; diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index a4e446750a7..82ff3aaba98 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -1,7 +1,9 @@ #include #include #include +#include #include +#include #include namespace DB @@ -9,7 +11,6 @@ namespace DB namespace ErrorCodes { extern const int INVALID_GRANT; - extern const int LOGICAL_ERROR; } @@ -58,12 +59,194 @@ namespace const AccessFlags system_reload_embedded_dictionaries = AccessType::SYSTEM_RELOAD_EMBEDDED_DICTIONARIES; }; - std::string_view checkCurrentDatabase(const std::string_view & current_database) + using Kind = AccessRightsElementWithOptions::Kind; + + struct ProtoElement { - if (current_database.empty()) - throw Exception("No current database", ErrorCodes::LOGICAL_ERROR); - return current_database; - } + AccessFlags access_flags; + boost::container::small_vector full_name; + bool grant_option = false; + Kind kind = Kind::GRANT; + + friend bool operator<(const ProtoElement & left, const ProtoElement & right) + { + static constexpr auto compare_name = [](const boost::container::small_vector & left_name, + const boost::container::small_vector & right_name, + size_t i) + { + if (i < left_name.size()) + { + if (i < right_name.size()) + return left_name[i].compare(right_name[i]); + else + return 1; /// left_name is longer => left_name > right_name + } + else if (i < right_name.size()) + return 1; /// right_name is longer => left < right + else + return 0; /// left_name == right_name + }; + + if (int cmp = compare_name(left.full_name, right.full_name, 0)) + return cmp < 0; + + if (int cmp = compare_name(left.full_name, right.full_name, 1)) + return cmp < 0; + + if (left.kind != right.kind) + return (left.kind == Kind::GRANT); + + if (left.grant_option != right.grant_option) + return right.grant_option; + + if (int cmp = compare_name(left.full_name, right.full_name, 2)) + return cmp < 0; + + return (left.access_flags < right.access_flags); + } + + AccessRightsElementWithOptions getResult() const + { + AccessRightsElementWithOptions res; + res.access_flags = access_flags; + res.grant_option = grant_option; + res.kind = kind; + switch (full_name.size()) + { + case 0: + { + res.any_database = true; + res.any_table = true; + res.any_column = true; + break; + } + case 1: + { + res.any_database = false; + res.database = full_name[0]; + res.any_table = true; + res.any_column = true; + break; + } + case 2: + { + res.any_database = false; + res.database = full_name[0]; + res.any_table = false; + res.table = full_name[1]; + res.any_column = true; + break; + } + case 3: + { + res.any_database = false; + res.database = full_name[0]; + res.any_table = false; + res.table = full_name[1]; + res.any_column = false; + res.columns.emplace_back(full_name[2]); + break; + } + } + return res; + } + }; + + class ProtoElements : public std::vector + { + public: + AccessRightsElementsWithOptions getResult() const + { + ProtoElements sorted = *this; + boost::range::sort(sorted); + AccessRightsElementsWithOptions res; + res.reserve(sorted.size()); + + for (size_t i = 0; i != sorted.size();) + { + size_t count_elements_with_diff_columns = sorted.countElementsWithDifferenceInColumnOnly(i); + if (count_elements_with_diff_columns == 1) + { + /// Easy case: one Element is converted to one AccessRightsElement. + const auto & element = sorted[i]; + if (element.access_flags) + res.emplace_back(element.getResult()); + ++i; + } + else + { + /// Difficult case: multiple Elements are converted to one or multiple AccessRightsElements. + sorted.appendResultWithElementsWithDifferenceInColumnOnly(i, count_elements_with_diff_columns, res); + i += count_elements_with_diff_columns; + } + } + return res; + } + + private: + size_t countElementsWithDifferenceInColumnOnly(size_t start) const + { + const auto & start_element = (*this)[start]; + if ((start_element.full_name.size() != 3) || (start == size() - 1)) + return 1; + + auto it = std::find_if(begin() + start + 1, end(), [&](const ProtoElement & element) + { + return (element.full_name.size() != 3) || (element.full_name[0] != start_element.full_name[0]) + || (element.full_name[1] != start_element.full_name[1]) || (element.grant_option != start_element.grant_option) + || (element.kind != start_element.kind); + }); + + return it - (begin() + start); + } + + /// Collects columns together to write multiple columns into one AccessRightsElement. + /// That procedure allows to output access rights in more compact way, + /// e.g. "SELECT(x, y)" instead of "SELECT(x), SELECT(y)". + void appendResultWithElementsWithDifferenceInColumnOnly(size_t start, size_t count, AccessRightsElementsWithOptions & res) const + { + const auto * pbegin = data() + start; + const auto * pend = pbegin + count; + AccessFlags handled_flags; + + while (pbegin < pend) + { + while (pbegin < pend && !(pbegin->access_flags - handled_flags)) + ++pbegin; + + while (pbegin < pend && !((pend - 1)->access_flags - handled_flags)) + --pend; + + if (pbegin >= pend) + break; + + AccessFlags common_flags = (pbegin->access_flags - handled_flags); + for (const auto * element = pbegin + 1; element != pend; ++element) + { + if (auto new_common_flags = (element->access_flags - handled_flags) & common_flags) + common_flags = new_common_flags; + } + + res.emplace_back(); + auto & back = res.back(); + back.grant_option = pbegin->grant_option; + back.kind = pbegin->kind; + back.any_database = false; + back.database = pbegin->full_name[0]; + back.any_table = false; + back.table = pbegin->full_name[1]; + back.any_column = false; + back.access_flags = common_flags; + for (const auto * element = pbegin; element != pend; ++element) + { + if (((element->access_flags - handled_flags) & common_flags) == common_flags) + back.columns.emplace_back(element->full_name[2]); + } + + handled_flags |= common_flags; + } + } + }; } @@ -249,17 +432,32 @@ public: calculateFinalAccessRec(helper); } - void logTree(Poco::Logger * log) const + + ProtoElements getElements() const { - LOG_TRACE(log, "Tree({}): name={}, access={}, final_access={}, min_access={}, max_access={}, num_children={}", - level, node_name ? *node_name : "NULL", access.toString(), + ProtoElements res; + getElementsRec(res, {}, *this, {}); + return res; + } + + static ProtoElements getElements(const Node * node, const Node * node_with_grant_option) + { + ProtoElements res; + getElementsRec(res, {}, node, {}, node_with_grant_option, {}); + return res; + } + + void logTree(Poco::Logger * log, const String & title) const + { + LOG_TRACE(log, "Tree({}): level={}, name={}, access={}, final_access={}, min_access={}, max_access={}, num_children={}", + title, level, node_name ? *node_name : "NULL", access.toString(), final_access.toString(), min_access.toString(), max_access.toString(), (children ? children->size() : 0)); if (children) { for (auto & child : *children | boost::adaptors::map_values) - child.logTree(log); + child.logTree(log, title); } } @@ -342,6 +540,93 @@ private: } } + static void getElementsRec( + ProtoElements & res, + const boost::container::small_vector & full_name, + const Node & node, + const AccessFlags & parent_access) + { + auto access = node.access; + auto revokes = parent_access - access; + auto grants = access - parent_access; + + if (revokes) + res.push_back(ProtoElement{revokes, full_name, false, Kind::REVOKE}); + + if (grants) + res.push_back(ProtoElement{grants, full_name, false, Kind::GRANT}); + + if (node.children) + { + for (const auto & [child_name, child] : *node.children) + { + boost::container::small_vector child_full_name = full_name; + child_full_name.push_back(child_name); + getElementsRec(res, child_full_name, child, access); + } + } + } + + static void getElementsRec( + ProtoElements & res, + const boost::container::small_vector & full_name, + const Node * node, + const AccessFlags & parent_access, + const Node * node_go, + const AccessFlags & parent_access_go) + { + auto access = node ? node->access : parent_access; + auto access_go = node_go ? node_go->access : parent_access_go; + auto revokes = parent_access - access; + auto revokes_go = parent_access_go - access_go - revokes; + auto grants_go = access_go - parent_access_go; + auto grants = access - parent_access - grants_go; + + if (revokes) + res.push_back(ProtoElement{revokes, full_name, false, Kind::REVOKE}); + + if (revokes_go) + res.push_back(ProtoElement{revokes_go, full_name, true, Kind::REVOKE}); + + if (grants) + res.push_back(ProtoElement{grants, full_name, false, Kind::GRANT}); + + if (grants_go) + res.push_back(ProtoElement{grants_go, full_name, true, Kind::GRANT}); + + if (node && node->children) + { + for (const auto & [child_name, child] : *node->children) + { + boost::container::small_vector child_full_name = full_name; + child_full_name.push_back(child_name); + const Node * child_node = &child; + const Node * child_node_go = nullptr; + if (node_go && node_go->children) + { + auto it = node_go->children->find(child_name); + if (it != node_go->children->end()) + child_node_go = &it->second; + } + getElementsRec(res, child_full_name, child_node, access, child_node_go, access_go); + } + + } + if (node_go && node_go->children) + { + for (const auto & [child_name, child] : *node_go->children) + { + if (node && node->children && node->children->count(child_name)) + continue; /// already processed + boost::container::small_vector child_full_name = full_name; + child_full_name.push_back(child_name); + const Node * child_node = nullptr; + const Node * child_node_go = &child; + getElementsRec(res, child_full_name, child_node, access, child_node_go, access_go); + } + } + } + void calculateFinalAccessRec(const Helper & helper) { /// Traverse tree. @@ -476,6 +761,10 @@ AccessRights & AccessRights::operator =(const AccessRights & src) root = std::make_unique(*src.root); else root = nullptr; + if (src.root_with_grant_option) + root_with_grant_option = std::make_unique(*src.root_with_grant_option); + else + root_with_grant_option = nullptr; return *this; } @@ -488,302 +777,245 @@ AccessRights::AccessRights(const AccessFlags & access) bool AccessRights::isEmpty() const { - return !root; + return !root && !root_with_grant_option; } void AccessRights::clear() { root = nullptr; + root_with_grant_option = nullptr; } -template +template void AccessRights::grantImpl(const AccessFlags & flags, const Args &... args) { - if (!root) - root = std::make_unique(); - root->grant(flags, Helper::instance(), args...); - if (!root->access && !root->children) - root = nullptr; + auto helper = [&](std::unique_ptr & root_node) + { + if (!root_node) + root_node = std::make_unique(); + root_node->grant(flags, Helper::instance(), args...); + if (!root_node->access && !root_node->children) + root_node = nullptr; + }; + helper(root); + + if constexpr (with_grant_option) + helper(root_with_grant_option); } -void AccessRights::grant(const AccessFlags & flags) { grantImpl(flags); } -void AccessRights::grant(const AccessFlags & flags, const std::string_view & database) { grantImpl(flags, database); } -void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl(flags, database, table); } -void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl(flags, database, table, column); } -void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) { grantImpl(flags, database, table, columns); } -void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl(flags, database, table, columns); } - -void AccessRights::grant(const AccessRightsElement & element, std::string_view current_database) +template +void AccessRights::grantImpl(const AccessRightsElement & element) { if (element.any_database) - { - grant(element.access_flags); - } + grantImpl(element.access_flags); else if (element.any_table) - { - if (element.database.empty()) - grant(element.access_flags, checkCurrentDatabase(current_database)); - else - grant(element.access_flags, element.database); - } + grantImpl(element.access_flags, element.database); else if (element.any_column) - { - if (element.database.empty()) - grant(element.access_flags, checkCurrentDatabase(current_database), element.table); - else - grant(element.access_flags, element.database, element.table); - } + grantImpl(element.access_flags, element.database, element.table); else - { - if (element.database.empty()) - grant(element.access_flags, checkCurrentDatabase(current_database), element.table, element.columns); - else - grant(element.access_flags, element.database, element.table, element.columns); - } + grantImpl(element.access_flags, element.database, element.table, element.columns); } -void AccessRights::grant(const AccessRightsElements & elements, std::string_view current_database) +template +void AccessRights::grantImpl(const AccessRightsElements & elements) { for (const auto & element : elements) - grant(element, current_database); + grantImpl(element); } +void AccessRights::grant(const AccessFlags & flags) { grantImpl(flags); } +void AccessRights::grant(const AccessFlags & flags, const std::string_view & database) { grantImpl(flags, database); } +void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl(flags, database, table); } +void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl(flags, database, table, column); } +void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) { grantImpl(flags, database, table, columns); } +void AccessRights::grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl(flags, database, table, columns); } +void AccessRights::grant(const AccessRightsElement & element) { grantImpl(element); } +void AccessRights::grant(const AccessRightsElements & elements) { grantImpl(elements); } -template +void AccessRights::grantWithGrantOption(const AccessFlags & flags) { grantImpl(flags); } +void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database) { grantImpl(flags, database); } +void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { grantImpl(flags, database, table); } +void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl(flags, database, table, column); } +void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) { grantImpl(flags, database, table, columns); } +void AccessRights::grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl(flags, database, table, columns); } +void AccessRights::grantWithGrantOption(const AccessRightsElement & element) { grantImpl(element); } +void AccessRights::grantWithGrantOption(const AccessRightsElements & elements) { grantImpl(elements); } + + +template void AccessRights::revokeImpl(const AccessFlags & flags, const Args &... args) { - if (!root) - return; - root->revoke(flags, Helper::instance(), args...); - if (!root->access && !root->children) - root = nullptr; + auto helper = [&](std::unique_ptr & root_node) + { + if (!root_node) + return; + root_node->revoke(flags, Helper::instance(), args...); + if (!root_node->access && !root_node->children) + root_node = nullptr; + }; + helper(root_with_grant_option); + + if constexpr (!grant_option) + helper(root); } -void AccessRights::revoke(const AccessFlags & flags) { revokeImpl(flags); } -void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database) { revokeImpl(flags, database); } -void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl(flags, database, table); } -void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl(flags, database, table, column); } -void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) { revokeImpl(flags, database, table, columns); } -void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl(flags, database, table, columns); } - - -void AccessRights::revoke(const AccessRightsElement & element, std::string_view current_database) +template +void AccessRights::revokeImpl(const AccessRightsElement & element) { if (element.any_database) - { - revoke(element.access_flags); - } + revokeImpl(element.access_flags); else if (element.any_table) - { - if (element.database.empty()) - revoke(element.access_flags, checkCurrentDatabase(current_database)); - else - revoke(element.access_flags, element.database); - } + revokeImpl(element.access_flags, element.database); else if (element.any_column) - { - if (element.database.empty()) - revoke(element.access_flags, checkCurrentDatabase(current_database), element.table); - else - revoke(element.access_flags, element.database, element.table); - } + revokeImpl(element.access_flags, element.database, element.table); else - { - if (element.database.empty()) - revoke(element.access_flags, checkCurrentDatabase(current_database), element.table, element.columns); - else - revoke(element.access_flags, element.database, element.table, element.columns); - } + revokeImpl(element.access_flags, element.database, element.table, element.columns); } -void AccessRights::revoke(const AccessRightsElements & elements, std::string_view current_database) +template +void AccessRights::revokeImpl(const AccessRightsElements & elements) { for (const auto & element : elements) - revoke(element, current_database); + revokeImpl(element); } +void AccessRights::revoke(const AccessFlags & flags) { revokeImpl(flags); } +void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database) { revokeImpl(flags, database); } +void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl(flags, database, table); } +void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl(flags, database, table, column); } +void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) { revokeImpl(flags, database, table, columns); } +void AccessRights::revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl(flags, database, table, columns); } +void AccessRights::revoke(const AccessRightsElement & element) { revokeImpl(element); } +void AccessRights::revoke(const AccessRightsElements & elements) { revokeImpl(elements); } -AccessRightsElements AccessRights::getGrants() const -{ - AccessRightsElements grants; - getGrantsAndPartialRevokesImpl(&grants, nullptr); - return grants; -} - -AccessRightsElements AccessRights::getPartialRevokes() const -{ - AccessRightsElements partial_revokes; - getGrantsAndPartialRevokesImpl(nullptr, &partial_revokes); - return partial_revokes; -} - -AccessRights::GrantsAndPartialRevokes AccessRights::getGrantsAndPartialRevokes() const -{ - GrantsAndPartialRevokes res; - getGrantsAndPartialRevokesImpl(&res.grants, &res.revokes); - return res; -} - - -void AccessRights::getGrantsAndPartialRevokesImpl(AccessRightsElements * out_grants, AccessRightsElements * out_partial_revokes) const +void AccessRights::revokeGrantOption(const AccessFlags & flags) { revokeImpl(flags); } +void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database) { revokeImpl(flags, database); } +void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) { revokeImpl(flags, database, table); } +void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl(flags, database, table, column); } +void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) { revokeImpl(flags, database, table, columns); } +void AccessRights::revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl(flags, database, table, columns); } +void AccessRights::revokeGrantOption(const AccessRightsElement & element) { revokeImpl(element); } +void AccessRights::revokeGrantOption(const AccessRightsElements & elements) { revokeImpl(elements); } + + +AccessRightsElementsWithOptions AccessRights::getElements() const { +#if 0 + logTree(); +#endif if (!root) - return; - auto global_access = root->access; - if (out_grants && global_access) - out_grants->push_back({global_access}); - if (root->children) - { - for (const auto & [db_name, db_node] : *root->children) - { - if (out_grants) - { - if (auto db_grants = db_node.access - global_access) - out_grants->push_back({db_grants, db_name}); - } - if (out_partial_revokes) - { - if (auto db_partial_revokes = global_access - db_node.access) - out_partial_revokes->push_back({db_partial_revokes, db_name}); - } - if (db_node.children) - { - for (const auto & [table_name, table_node] : *db_node.children) - { - if (out_grants) - { - if (auto table_grants = table_node.access - db_node.access) - out_grants->push_back({table_grants, db_name, table_name}); - } - if (out_partial_revokes) - { - if (auto table_partial_revokes = db_node.access - table_node.access) - out_partial_revokes->push_back({table_partial_revokes, db_name, table_name}); - } - if (table_node.children) - { - for (const auto & [column_name, column_node] : *table_node.children) - { - if (out_grants) - { - if (auto column_grants = column_node.access - table_node.access) - out_grants->push_back({column_grants, db_name, table_name, column_name}); - } - if (out_partial_revokes) - { - if (auto column_partial_revokes = table_node.access - column_node.access) - out_partial_revokes->push_back({column_partial_revokes, db_name, table_name, column_name}); - } - } - - } - } - } - } - } + return {}; + if (!root_with_grant_option) + return root->getElements().getResult(); + return Node::getElements(root.get(), root_with_grant_option.get()).getResult(); } String AccessRights::toString() const { - String res; - auto gr = getGrantsAndPartialRevokes(); - if (!gr.grants.empty()) - { - res += "GRANT "; - res += gr.grants.toString(); - } - if (!gr.revokes.empty()) - { - if (!res.empty()) - res += ", "; - res += "REVOKE "; - res += gr.revokes.toString(); - } - if (res.empty()) - res = "GRANT USAGE ON *.*"; - return res; + return getElements().toString(); } -template +template bool AccessRights::isGrantedImpl(const AccessFlags & flags, const Args &... args) const { - if (!root) - return flags.isEmpty(); - return root->isGranted(flags, args...); + auto helper = [&](const std::unique_ptr & root_node) -> bool + { + if (!root_node) + return flags.isEmpty(); + return root_node->isGranted(flags, args...); + }; + if constexpr (grant_option) + return helper(root_with_grant_option); + else + return helper(root); } -bool AccessRights::isGranted(const AccessFlags & flags) const { return isGrantedImpl(flags); } -bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl(flags, database); } -bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl(flags, database, table); } -bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl(flags, database, table, column); } -bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { return isGrantedImpl(flags, database, table, columns); } -bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl(flags, database, table, columns); } - -bool AccessRights::isGranted(const AccessRightsElement & element, std::string_view current_database) const +template +bool AccessRights::isGrantedImpl(const AccessRightsElement & element) const { if (element.any_database) - { - return isGranted(element.access_flags); - } + return isGrantedImpl(element.access_flags); else if (element.any_table) - { - if (element.database.empty()) - return isGranted(element.access_flags, checkCurrentDatabase(current_database)); - else - return isGranted(element.access_flags, element.database); - } + return isGrantedImpl(element.access_flags, element.database); else if (element.any_column) - { - if (element.database.empty()) - return isGranted(element.access_flags, checkCurrentDatabase(current_database), element.table); - else - return isGranted(element.access_flags, element.database, element.table); - } + return isGrantedImpl(element.access_flags, element.database, element.table); else - { - if (element.database.empty()) - return isGranted(element.access_flags, checkCurrentDatabase(current_database), element.table, element.columns); - else - return isGranted(element.access_flags, element.database, element.table, element.columns); - } + return isGrantedImpl(element.access_flags, element.database, element.table, element.columns); } -bool AccessRights::isGranted(const AccessRightsElements & elements, std::string_view current_database) const +template +bool AccessRights::isGrantedImpl(const AccessRightsElements & elements) const { for (const auto & element : elements) - if (!isGranted(element, current_database)) + if (!isGrantedImpl(element)) return false; return true; } +bool AccessRights::isGranted(const AccessFlags & flags) const { return isGrantedImpl(flags); } +bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl(flags, database); } +bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl(flags, database, table); } +bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl(flags, database, table, column); } +bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { return isGrantedImpl(flags, database, table, columns); } +bool AccessRights::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl(flags, database, table, columns); } +bool AccessRights::isGranted(const AccessRightsElement & element) const { return isGrantedImpl(element); } +bool AccessRights::isGranted(const AccessRightsElements & elements) const { return isGrantedImpl(elements); } + +bool AccessRights::hasGrantOption(const AccessFlags & flags) const { return isGrantedImpl(flags); } +bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl(flags, database); } +bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl(flags, database, table); } +bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl(flags, database, table, column); } +bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { return isGrantedImpl(flags, database, table, columns); } +bool AccessRights::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl(flags, database, table, columns); } +bool AccessRights::hasGrantOption(const AccessRightsElement & element) const { return isGrantedImpl(element); } +bool AccessRights::hasGrantOption(const AccessRightsElements & elements) const { return isGrantedImpl(elements); } + bool operator ==(const AccessRights & left, const AccessRights & right) { - if (!left.root) - return !right.root; - if (!right.root) - return false; - return *left.root == *right.root; + auto helper = [](const std::unique_ptr & left_node, const std::unique_ptr & right_node) + { + if (!left_node) + return !right_node; + if (!right_node) + return false; + return *left_node == *right_node; + }; + return helper(left.root, right.root) && helper(left.root_with_grant_option, right.root_with_grant_option); } void AccessRights::merge(const AccessRights & other) { - if (!root) + auto helper = [](std::unique_ptr & root_node, const std::unique_ptr & other_root_node) { - *this = other; - return; - } - if (other.root) - { - root->merge(*other.root, Helper::instance()); - if (!root->access && !root->children) - root = nullptr; - } + if (!root_node) + { + if (other_root_node) + root_node = std::make_unique(*other_root_node); + return; + } + if (other_root_node) + { + root_node->merge(*other_root_node, Helper::instance()); + if (!root_node->access && !root_node->children) + root_node = nullptr; + } + }; + helper(root, other.root); + helper(root_with_grant_option, other.root_with_grant_option); +} + + +AccessRights AccessRights::getFullAccess() +{ + AccessRights res; + res.grantWithGrantOption(AccessType::ALL); + return res; } @@ -791,7 +1023,11 @@ void AccessRights::logTree() const { auto * log = &Poco::Logger::get("AccessRights"); if (root) - root->logTree(log); + { + root->logTree(log, ""); + if (root_with_grant_option) + root->logTree(log, "go"); + } else LOG_TRACE(log, "Tree: NULL"); } diff --git a/src/Access/AccessRights.h b/src/Access/AccessRights.h index c32514e8feb..7706edcb40a 100644 --- a/src/Access/AccessRights.h +++ b/src/Access/AccessRights.h @@ -26,6 +26,12 @@ public: /// Revokes everything. It's the same as revoke(AccessType::ALL). void clear(); + /// Returns the information about all the access granted as a string. + String toString() const; + + /// Returns the information about all the access granted. + AccessRightsElementsWithOptions getElements() const; + /// Grants access on a specified database/table/column. /// Does nothing if the specified access has been already granted. void grant(const AccessFlags & flags); @@ -34,8 +40,17 @@ public: void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column); void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns); void grant(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns); - void grant(const AccessRightsElement & element, std::string_view current_database = {}); - void grant(const AccessRightsElements & elements, std::string_view current_database = {}); + void grant(const AccessRightsElement & element); + void grant(const AccessRightsElements & elements); + + void grantWithGrantOption(const AccessFlags & flags); + void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database); + void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table); + void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column); + void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns); + void grantWithGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns); + void grantWithGrantOption(const AccessRightsElement & element); + void grantWithGrantOption(const AccessRightsElements & elements); /// Revokes a specified access granted earlier on a specified database/table/column. /// For example, revoke(AccessType::ALL) revokes all grants at all, just like clear(); @@ -45,21 +60,17 @@ public: void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column); void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns); void revoke(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns); - void revoke(const AccessRightsElement & element, std::string_view current_database = {}); - void revoke(const AccessRightsElements & elements, std::string_view current_database = {}); + void revoke(const AccessRightsElement & element); + void revoke(const AccessRightsElements & elements); - /// Returns the information about all the access granted. - struct GrantsAndPartialRevokes - { - AccessRightsElements grants; - AccessRightsElements revokes; - }; - AccessRightsElements getGrants() const; - AccessRightsElements getPartialRevokes() const; - GrantsAndPartialRevokes getGrantsAndPartialRevokes() const; - - /// Returns the information about all the access granted as a string. - String toString() const; + void revokeGrantOption(const AccessFlags & flags); + void revokeGrantOption(const AccessFlags & flags, const std::string_view & database); + void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table); + void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column); + void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns); + void revokeGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns); + void revokeGrantOption(const AccessRightsElement & element); + void revokeGrantOption(const AccessRightsElements & elements); /// Whether a specified access granted. bool isGranted(const AccessFlags & flags) const; @@ -68,38 +79,60 @@ public: bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; - bool isGranted(const AccessRightsElement & element, std::string_view current_database = {}) const; - bool isGranted(const AccessRightsElements & elements, std::string_view current_database = {}) const; + bool isGranted(const AccessRightsElement & element) const; + bool isGranted(const AccessRightsElements & elements) const; - friend bool operator ==(const AccessRights & left, const AccessRights & right); - friend bool operator !=(const AccessRights & left, const AccessRights & right) { return !(left == right); } + bool hasGrantOption(const AccessFlags & flags) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; + bool hasGrantOption(const AccessRightsElement & element) const; + bool hasGrantOption(const AccessRightsElements & elements) const; /// Merges two sets of access rights together. /// It's used to combine access rights from multiple roles. void merge(const AccessRights & other); + friend bool operator ==(const AccessRights & left, const AccessRights & right); + friend bool operator !=(const AccessRights & left, const AccessRights & right) { return !(left == right); } + + static AccessRights getFullAccess(); + private: - template + template void grantImpl(const AccessFlags & flags, const Args &... args); - template + template + void grantImpl(const AccessRightsElement & element); + + template + void grantImpl(const AccessRightsElements & elements); + + template void revokeImpl(const AccessFlags & flags, const Args &... args); - template + template + void revokeImpl(const AccessRightsElement & element); + + template + void revokeImpl(const AccessRightsElements & elements); + + template bool isGrantedImpl(const AccessFlags & flags, const Args &... args) const; - bool isGrantedImpl(const AccessRightsElement & element, std::string_view current_database) const; - bool isGrantedImpl(const AccessRightsElements & elements, std::string_view current_database) const; + template + bool isGrantedImpl(const AccessRightsElement & element) const; - template - AccessFlags getAccessImpl(const Args &... args) const; - - void getGrantsAndPartialRevokesImpl(AccessRightsElements * grants, AccessRightsElements * partial_revokes) const; + template + bool isGrantedImpl(const AccessRightsElements & elements) const; void logTree() const; struct Node; std::unique_ptr root; + std::unique_ptr root_with_grant_option; }; } diff --git a/src/Access/AccessRightsElement.cpp b/src/Access/AccessRightsElement.cpp index db1ea5d3d5c..e69fb6d3b74 100644 --- a/src/Access/AccessRightsElement.cpp +++ b/src/Access/AccessRightsElement.cpp @@ -12,222 +12,158 @@ namespace DB { namespace { - size_t groupElements(AccessRightsElements & elements, size_t start) + using Kind = AccessRightsElementWithOptions::Kind; + + String formatOptions(bool grant_option, Kind kind, const String & inner_part) { - auto & start_element = elements[start]; - auto it = std::find_if(elements.begin() + start + 1, elements.end(), - [&](const AccessRightsElement & element) + if (kind == Kind::REVOKE) { - return (element.database != start_element.database) || - (element.any_database != start_element.any_database) || - (element.table != start_element.table) || - (element.any_table != start_element.any_table) || - (element.any_column != start_element.any_column); - }); - size_t end = it - elements.begin(); - - /// All the elements at indices from start to end here specify - /// the same database and table. - - if (start_element.any_column) - { - /// Easy case: the elements don't specify columns. - /// All we need is to combine the access flags. - for (size_t i = start + 1; i != end; ++i) - { - start_element.access_flags |= elements[i].access_flags; - elements[i].access_flags = {}; - } - return end; + if (grant_option) + return "REVOKE GRANT OPTION " + inner_part; + else + return "REVOKE " + inner_part; } - - /// Difficult case: the elements specify columns. - /// We have to find groups of columns with common access flags. - for (size_t i = start; i != end; ++i) + else { - if (!elements[i].access_flags) - continue; - - AccessFlags common_flags = elements[i].access_flags; - size_t num_elements_with_common_flags = 1; - for (size_t j = i + 1; j != end; ++j) - { - auto new_common_flags = common_flags & elements[j].access_flags; - if (new_common_flags) - { - common_flags = new_common_flags; - ++num_elements_with_common_flags; - } - } - - if (num_elements_with_common_flags == 1) - continue; - - if (elements[i].access_flags != common_flags) - { - elements.insert(elements.begin() + i + 1, elements[i]); - elements[i].access_flags = common_flags; - elements[i].columns.clear(); - ++end; - } - - for (size_t j = i + 1; j != end; ++j) - { - if ((elements[j].access_flags & common_flags) == common_flags) - { - boost::range::push_back(elements[i].columns, elements[j].columns); - elements[j].access_flags -= common_flags; - } - } + if (grant_option) + return "GRANT " + inner_part + " WITH GRANT OPTION"; + else + return "GRANT " + inner_part; } - - return end; } - /// Tries to combine elements to decrease their number. - void groupElements(AccessRightsElements & elements) + + String formatONClause(const String & database, bool any_database, const String & table, bool any_table) { - if (!boost::range::is_sorted(elements)) - boost::range::sort(elements); /// Algorithm in groupElement() requires elements to be sorted. - for (size_t start = 0; start != elements.size();) - start = groupElements(elements, start); + String msg = "ON "; + + if (any_database) + msg += "*."; + else if (!database.empty()) + msg += backQuoteIfNeed(database) + "."; + + if (any_table) + msg += "*"; + else + msg += backQuoteIfNeed(table); + return msg; } - /// Removes unnecessary elements, sorts elements and makes them unique. - void sortElementsAndMakeUnique(AccessRightsElements & elements) + + String formatAccessFlagsWithColumns(const AccessFlags & access_flags, const Strings & columns, bool any_column) { - /// Remove empty elements. - boost::range::remove_erase_if(elements, [](const AccessRightsElement & element) + String columns_in_parentheses; + if (!any_column) { - return !element.access_flags || (!element.any_column && element.columns.empty()); - }); - - /// Sort columns and make them unique. - for (auto & element : elements) - { - if (element.any_column) - continue; - - if (!boost::range::is_sorted(element.columns)) - boost::range::sort(element.columns); - element.columns.erase(std::unique(element.columns.begin(), element.columns.end()), element.columns.end()); + if (columns.empty()) + return "USAGE"; + for (const auto & column : columns) + { + columns_in_parentheses += columns_in_parentheses.empty() ? "(" : ", "; + columns_in_parentheses += backQuoteIfNeed(column); + } + columns_in_parentheses += ")"; } - /// Sort elements themselves. - boost::range::sort(elements); - elements.erase(std::unique(elements.begin(), elements.end()), elements.end()); + auto keywords = access_flags.toKeywords(); + if (keywords.empty()) + return "USAGE"; + + String msg; + for (const std::string_view & keyword : keywords) + { + if (!msg.empty()) + msg += ", "; + msg += String{keyword} + columns_in_parentheses; + } + return msg; } } -void AccessRightsElement::setDatabase(const String & new_database) -{ - database = new_database; - any_database = false; -} - - -void AccessRightsElement::replaceEmptyDatabase(const String & new_database) -{ - if (isEmptyDatabase()) - setDatabase(new_database); -} - - -bool AccessRightsElement::isEmptyDatabase() const -{ - return !any_database && database.empty(); -} - String AccessRightsElement::toString() const { - String msg = toStringWithoutON(); - msg += " ON "; - - if (any_database) - msg += "*."; - else if (!database.empty()) - msg += backQuoteIfNeed(database) + "."; - - if (any_table) - msg += "*"; - else - msg += backQuoteIfNeed(table); - return msg; + return formatAccessFlagsWithColumns(access_flags, columns, any_column) + " " + formatONClause(database, any_database, table, any_table); } -String AccessRightsElement::toStringWithoutON() const +String AccessRightsElementWithOptions::toString() const { - String columns_in_parentheses; - if (!any_column) - { - if (columns.empty()) - return "USAGE"; - for (const auto & column : columns) - { - columns_in_parentheses += columns_in_parentheses.empty() ? "(" : ", "; - columns_in_parentheses += backQuoteIfNeed(column); - } - columns_in_parentheses += ")"; - } - - auto keywords = access_flags.toKeywords(); - if (keywords.empty()) - return "USAGE"; - - String msg; - for (const std::string_view & keyword : keywords) - { - if (!msg.empty()) - msg += ", "; - msg += String{keyword} + columns_in_parentheses; - } - return msg; + return formatOptions(grant_option, kind, AccessRightsElement::toString()); } - -void AccessRightsElements::replaceEmptyDatabase(const String & new_database) +String AccessRightsElements::toString() const { - for (auto & element : *this) - element.replaceEmptyDatabase(new_database); -} - - -String AccessRightsElements::toString() -{ - normalize(); - if (empty()) return "USAGE ON *.*"; - String msg; - bool need_comma = false; + String res; + String inner_part; + for (size_t i = 0; i != size(); ++i) { const auto & element = (*this)[i]; - if (std::exchange(need_comma, true)) - msg += ", "; - bool next_element_on_same_db_and_table = false; + + if (!inner_part.empty()) + inner_part += ", "; + inner_part += formatAccessFlagsWithColumns(element.access_flags, element.columns, element.any_column); + + bool next_element_uses_same_table = false; if (i != size() - 1) { const auto & next_element = (*this)[i + 1]; - if ((element.database == next_element.database) && (element.any_database == next_element.any_database) - && (element.table == next_element.table) && (element.any_table == next_element.any_table)) - next_element_on_same_db_and_table = true; + if (element.sameDatabaseAndTable(next_element)) + next_element_uses_same_table = true; + } + + if (!next_element_uses_same_table) + { + if (!res.empty()) + res += ", "; + res += inner_part + " " + formatONClause(element.database, element.any_database, element.table, element.any_table); + inner_part.clear(); } - if (next_element_on_same_db_and_table) - msg += element.toStringWithoutON(); - else - msg += element.toString(); } - return msg; + + return res; } - -void AccessRightsElements::normalize() +String AccessRightsElementsWithOptions::toString() const { - groupElements(*this); - sortElementsAndMakeUnique(*this); + if (empty()) + return "GRANT USAGE ON *.*"; + + String res; + String inner_part; + + for (size_t i = 0; i != size(); ++i) + { + const auto & element = (*this)[i]; + + if (!inner_part.empty()) + inner_part += ", "; + inner_part += formatAccessFlagsWithColumns(element.access_flags, element.columns, element.any_column); + + bool next_element_uses_same_mode_and_table = false; + if (i != size() - 1) + { + const auto & next_element = (*this)[i + 1]; + if (element.sameDatabaseAndTable(next_element) && element.sameOptions(next_element)) + next_element_uses_same_mode_and_table = true; + } + + if (!next_element_uses_same_mode_and_table) + { + if (!res.empty()) + res += ", "; + res += formatOptions( + element.grant_option, + element.kind, + inner_part + " " + formatONClause(element.database, element.any_database, element.table, element.any_table)); + inner_part.clear(); + } + } + + return res; } } diff --git a/src/Access/AccessRightsElement.h b/src/Access/AccessRightsElement.h index 70eb95c2d17..f9f7c433308 100644 --- a/src/Access/AccessRightsElement.h +++ b/src/Access/AccessRightsElement.h @@ -71,26 +71,48 @@ struct AccessRightsElement { } - auto toTuple() const { return std::tie(access_flags, database, any_database, table, any_table, columns, any_column); } + auto toTuple() const { return std::tie(access_flags, any_database, database, any_table, table, any_column, columns); } friend bool operator==(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() == right.toTuple(); } - friend bool operator!=(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() != right.toTuple(); } - friend bool operator<(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() < right.toTuple(); } - friend bool operator>(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() > right.toTuple(); } - friend bool operator<=(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() <= right.toTuple(); } - friend bool operator>=(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() >= right.toTuple(); } + friend bool operator!=(const AccessRightsElement & left, const AccessRightsElement & right) { return !(left == right); } - /// Sets the database. - void setDatabase(const String & new_database); + bool sameDatabaseAndTable(const AccessRightsElement & other) const + { + return (database == other.database) && (any_database == other.any_database) && (table == other.table) + && (any_table == other.any_table); + } + + bool isEmptyDatabase() const { return !any_database && database.empty(); } /// If the database is empty, replaces it with `new_database`. Otherwise does nothing. void replaceEmptyDatabase(const String & new_database); - bool isEmptyDatabase() const; - /// Returns a human-readable representation like "SELECT, UPDATE(x, y) ON db.table". - /// The returned string isn't prefixed with the "GRANT" keyword. String toString() const; - String toStringWithoutON() const; +}; + + +struct AccessRightsElementWithOptions : public AccessRightsElement +{ + bool grant_option = false; + + enum class Kind + { + GRANT, + REVOKE, + }; + Kind kind = Kind::GRANT; + + bool sameOptions(const AccessRightsElementWithOptions & other) const + { + return (grant_option == other.grant_option) && (kind == other.kind); + } + + auto toTuple() const { return std::tie(access_flags, any_database, database, any_table, table, any_column, columns, grant_option, kind); } + friend bool operator==(const AccessRightsElementWithOptions & left, const AccessRightsElementWithOptions & right) { return left.toTuple() == right.toTuple(); } + friend bool operator!=(const AccessRightsElementWithOptions & left, const AccessRightsElementWithOptions & right) { return !(left == right); } + + /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table". + String toString() const; }; @@ -101,13 +123,38 @@ public: /// Replaces the empty database with `new_database`. void replaceEmptyDatabase(const String & new_database); - /// Returns a human-readable representation like "SELECT, UPDATE(x, y) ON db.table". - /// The returned string isn't prefixed with the "GRANT" keyword. - String toString() const { return AccessRightsElements(*this).toString(); } - String toString(); - - /// Reorder and group elements to show them in more readable form. - void normalize(); + /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table". + String toString() const; }; + +class AccessRightsElementsWithOptions : public std::vector +{ +public: + /// Replaces the empty database with `new_database`. + void replaceEmptyDatabase(const String & new_database); + + /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table". + String toString() const; +}; + + +inline void AccessRightsElement::replaceEmptyDatabase(const String & new_database) +{ + if (isEmptyDatabase()) + database = new_database; +} + +inline void AccessRightsElements::replaceEmptyDatabase(const String & new_database) +{ + for (auto & element : *this) + element.replaceEmptyDatabase(new_database); +} + +inline void AccessRightsElementsWithOptions::replaceEmptyDatabase(const String & new_database) +{ + for (auto & element : *this) + element.replaceEmptyDatabase(new_database); +} + } diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 62aebfd4367..4a156c5972d 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -15,8 +15,6 @@ #include #include #include -#include -#include #include #include @@ -32,68 +30,6 @@ namespace ErrorCodes extern const int UNKNOWN_USER; } - -namespace -{ - enum CheckAccessRightsMode - { - RETURN_FALSE_IF_ACCESS_DENIED, - LOG_WARNING_IF_ACCESS_DENIED, - THROW_IF_ACCESS_DENIED, - }; - - - String formatSkippedMessage() - { - return ""; - } - - String formatSkippedMessage(const std::string_view & database) - { - return ". Skipped database " + backQuoteIfNeed(database); - } - - String formatSkippedMessage(const std::string_view & database, const std::string_view & table) - { - String str = ". Skipped table "; - if (!database.empty()) - str += backQuoteIfNeed(database) + "."; - str += backQuoteIfNeed(table); - return str; - } - - String formatSkippedMessage(const std::string_view & database, const std::string_view & table, const std::string_view & column) - { - String str = ". Skipped column " + backQuoteIfNeed(column) + " ON "; - if (!database.empty()) - str += backQuoteIfNeed(database) + "."; - str += backQuoteIfNeed(table); - return str; - } - - template - String formatSkippedMessage(const std::string_view & database, const std::string_view & table, const std::vector & columns) - { - if (columns.size() == 1) - return formatSkippedMessage(database, table, columns[0]); - - String str = ". Skipped columns "; - bool need_comma = false; - for (const auto & column : columns) - { - if (std::exchange(need_comma, true)) - str += ", "; - str += backQuoteIfNeed(column); - } - str += " ON "; - if (!database.empty()) - str += backQuoteIfNeed(database) + "."; - str += backQuoteIfNeed(table); - return str; - } -} - - ContextAccess::ContextAccess(const AccessControlManager & manager_, const Params & params_) : manager(&manager_) , params(params_) @@ -116,8 +52,8 @@ void ContextAccess::setUser(const UserPtr & user_) const if (!user) { /// User has been dropped. - auto nothing_granted = boost::make_shared(); - boost::range::fill(result_access, nothing_granted); + auto nothing_granted = std::make_shared(); + access = nothing_granted; subscription_for_user_change = {}; subscription_for_roles_changes = {}; enabled_roles = nullptr; @@ -169,10 +105,73 @@ void ContextAccess::setRolesInfo(const std::shared_ptr & { assert(roles_info_); roles_info = roles_info_; - boost::range::fill(result_access, nullptr /* need recalculate */); enabled_row_policies = manager->getEnabledRowPolicies(*params.user_id, roles_info->enabled_roles); enabled_quota = manager->getEnabledQuota(*params.user_id, user_name, roles_info->enabled_roles, params.address, params.quota_key); enabled_settings = manager->getEnabledSettings(*params.user_id, user->settings, roles_info->enabled_roles, roles_info->settings_from_enabled_roles); + setFinalAccess(); +} + + +void ContextAccess::setFinalAccess() const +{ + auto final_access = std::make_shared(); + *final_access = user->access; + if (roles_info) + final_access->merge(roles_info->access); + + static const AccessFlags table_ddl = AccessType::CREATE_DATABASE | AccessType::CREATE_TABLE | AccessType::CREATE_VIEW + | AccessType::ALTER_TABLE | AccessType::ALTER_VIEW | AccessType::DROP_DATABASE | AccessType::DROP_TABLE | AccessType::DROP_VIEW + | AccessType::TRUNCATE; + + static const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY; + static const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl; + static const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE; + static const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS; + + if (params.readonly) + final_access->revoke(write_table_access | table_and_dictionary_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY); + + if (params.readonly == 1) + { + /// Table functions are forbidden in readonly mode. + /// For example, for readonly = 2 - allowed. + final_access->revoke(AccessType::CREATE_TEMPORARY_TABLE); + } + + if (!params.allow_ddl) + final_access->revoke(table_and_dictionary_ddl); + + if (!params.allow_introspection) + final_access->revoke(AccessType::INTROSPECTION); + + /// Anyone has access to the "system" database. + final_access->grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE); + + if (params.readonly != 1) + { + /// User has access to temporary or external table if such table was resolved in session or query context + final_access->grant(AccessFlags::allTableFlags() | AccessFlags::allColumnFlags(), DatabaseCatalog::TEMPORARY_DATABASE); + } + + if (params.readonly) + { + /// No grant option in readonly mode. + final_access->revokeGrantOption(AccessType::ALL); + } + + if (trace_log) + { + if (roles_info && !roles_info->getCurrentRolesNames().empty()) + { + LOG_TRACE(trace_log, "Current_roles: {}, enabled_roles: {}", + boost::algorithm::join(roles_info->getCurrentRolesNames(), ", "), + boost::algorithm::join(roles_info->getEnabledRolesNames(), ", ")); + } + LOG_TRACE(trace_log, "Settings: readonly={}, allow_ddl={}, allow_introspection_functions={}", params.readonly, params.allow_ddl, params.allow_introspection); + LOG_TRACE(trace_log, "List of all grants: {}", final_access->toString()); + } + + access = final_access; } @@ -193,284 +192,6 @@ bool ContextAccess::isClientHostAllowed() const } -template -bool ContextAccess::calculateResultAccessAndCheck(Poco::Logger * log_, const AccessFlags & flags, const Args &... args) const -{ - auto access = calculateResultAccess(grant_option); - bool is_granted = access->isGranted(flags, args...); - - if (trace_log) - LOG_TRACE(trace_log, "Access {}: {}", (is_granted ? "granted" : "denied"), (AccessRightsElement{flags, args...}.toString())); - - if (is_granted) - return true; - - if constexpr (mode == RETURN_FALSE_IF_ACCESS_DENIED) - return false; - - if constexpr (mode == LOG_WARNING_IF_ACCESS_DENIED) - { - if (!log_) - return false; - } - - auto show_error = [&](const String & msg, [[maybe_unused]] int error_code) - { - if constexpr (mode == THROW_IF_ACCESS_DENIED) - throw Exception(user_name + ": " + msg, error_code); - else if constexpr (mode == LOG_WARNING_IF_ACCESS_DENIED) - LOG_WARNING(log_, "{}: {}{}", user_name, msg, formatSkippedMessage(args...)); - }; - - if (!user) - { - show_error("User has been dropped", ErrorCodes::UNKNOWN_USER); - } - else if (grant_option && calculateResultAccess(false, params.readonly, params.allow_ddl, params.allow_introspection)->isGranted(flags, args...)) - { - show_error( - "Not enough privileges. " - "The required privileges have been granted, but without grant option. " - "To execute this query it's necessary to have the grant " - + AccessRightsElement{flags, args...}.toString() + " WITH GRANT OPTION", - ErrorCodes::ACCESS_DENIED); - } - else if (params.readonly && calculateResultAccess(false, false, params.allow_ddl, params.allow_introspection)->isGranted(flags, args...)) - { - if (params.interface == ClientInfo::Interface::HTTP && params.http_method == ClientInfo::HTTPMethod::GET) - show_error( - "Cannot execute query in readonly mode. " - "For queries over HTTP, method GET implies readonly. You should use method POST for modifying queries", - ErrorCodes::READONLY); - else - show_error("Cannot execute query in readonly mode", ErrorCodes::READONLY); - } - else if (!params.allow_ddl && calculateResultAccess(false, params.readonly, true, params.allow_introspection)->isGranted(flags, args...)) - { - show_error("Cannot execute query. DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED); - } - else if (!params.allow_introspection && calculateResultAccess(false, params.readonly, params.allow_ddl, true)->isGranted(flags, args...)) - { - show_error("Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED); - } - else - { - show_error( - "Not enough privileges. To execute this query it's necessary to have the grant " - + AccessRightsElement{flags, args...}.toString() + (grant_option ? " WITH GRANT OPTION" : ""), - ErrorCodes::ACCESS_DENIED); - } - - return false; -} - - -template -bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags) const -{ - return calculateResultAccessAndCheck(log_, flags); -} - -template -bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const Args &... args) const -{ - if (database.empty()) - return calculateResultAccessAndCheck(log_, flags, params.current_database, args...); - else - return calculateResultAccessAndCheck(log_, flags, database, args...); -} - - -template -bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessRightsElement & element) const -{ - if (element.any_database) - { - return checkAccessImpl(log_, element.access_flags); - } - else if (element.any_table) - { - return checkAccessImpl(log_, element.access_flags, element.database); - } - else if (element.any_column) - { - return checkAccessImpl(log_, element.access_flags, element.database, element.table); - } - else - { - return checkAccessImpl(log_, element.access_flags, element.database, element.table, element.columns); - } -} - - -template -bool ContextAccess::checkAccessImpl(Poco::Logger * log_, const AccessRightsElements & elements) const -{ - for (const auto & element : elements) - if (!checkAccessImpl(log_, element)) - return false; - return true; -} - - -void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl(nullptr, flags); } -void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl(nullptr, flags, database); } -void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl(nullptr, flags, database, table); } -void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl(nullptr, flags, database, table, column); } -void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { checkAccessImpl(nullptr, flags, database, table, columns); } -void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl(nullptr, flags, database, table, columns); } -void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl(nullptr, element); } -void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl(nullptr, elements); } - -bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl(nullptr, flags); } -bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl(nullptr, flags, database); } -bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl(nullptr, flags, database, table); } -bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl(nullptr, flags, database, table, column); } -bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { return checkAccessImpl(nullptr, flags, database, table, columns); } -bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl(nullptr, flags, database, table, columns); } -bool ContextAccess::isGranted(const AccessRightsElement & element) const { return checkAccessImpl(nullptr, element); } -bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return checkAccessImpl(nullptr, elements); } - -bool ContextAccess::isGranted(Poco::Logger * log_, const AccessFlags & flags) const { return checkAccessImpl(log_, flags); } -bool ContextAccess::isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl(log_, flags, database); } -bool ContextAccess::isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return checkAccessImpl(log_, flags, database, table); } -bool ContextAccess::isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkAccessImpl(log_, flags, database, table, column); } -bool ContextAccess::isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { return checkAccessImpl(log_, flags, database, table, columns); } -bool ContextAccess::isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkAccessImpl(log_, flags, database, table, columns); } -bool ContextAccess::isGranted(Poco::Logger * log_, const AccessRightsElement & element) const { return checkAccessImpl(log_, element); } -bool ContextAccess::isGranted(Poco::Logger * log_, const AccessRightsElements & elements) const { return checkAccessImpl(log_, elements); } - -void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl(nullptr, flags); } -void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl(nullptr, flags, database); } -void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl(nullptr, flags, database, table); } -void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl(nullptr, flags, database, table, column); } -void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { checkAccessImpl(nullptr, flags, database, table, columns); } -void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl(nullptr, flags, database, table, columns); } -void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl(nullptr, element); } -void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl(nullptr, elements); } - - -void ContextAccess::checkAdminOption(const UUID & role_id) const -{ - if (isGranted(AccessType::ROLE_ADMIN)) - return; - - auto info = getRolesInfo(); - if (info && info->enabled_roles_with_admin_option.count(role_id)) - return; - - if (!user) - throw Exception(user_name + ": User has been dropped", ErrorCodes::UNKNOWN_USER); - - std::optional role_name = manager->readName(role_id); - if (!role_name) - role_name = "ID {" + toString(role_id) + "}"; - throw Exception( - getUserName() + ": Not enough privileges. To execute this query it's necessary to have the grant " + backQuoteIfNeed(*role_name) - + " WITH ADMIN OPTION ", - ErrorCodes::ACCESS_DENIED); -} - - -boost::shared_ptr ContextAccess::calculateResultAccess(bool grant_option) const -{ - return calculateResultAccess(grant_option, params.readonly, params.allow_ddl, params.allow_introspection); -} - - -boost::shared_ptr ContextAccess::calculateResultAccess(bool grant_option, UInt64 readonly_, bool allow_ddl_, bool allow_introspection_) const -{ - size_t index = static_cast(readonly_ != params.readonly) - + static_cast(allow_ddl_ != params.allow_ddl) * 2 + - + static_cast(allow_introspection_ != params.allow_introspection) * 3 - + static_cast(grant_option) * 4; - assert(index < std::size(result_access)); - auto res = result_access[index].load(); - if (res) - return res; - - std::lock_guard lock{mutex}; - res = result_access[index].load(); - if (res) - return res; - - auto merged_access = boost::make_shared(); - - if (grant_option) - { - *merged_access = user->access.access_with_grant_option; - if (roles_info) - merged_access->merge(roles_info->access_with_grant_option); - } - else - { - *merged_access = user->access.access; - if (roles_info) - merged_access->merge(roles_info->access); - } - - static const AccessFlags table_ddl = AccessType::CREATE_DATABASE | AccessType::CREATE_TABLE | AccessType::CREATE_VIEW - | AccessType::ALTER_TABLE | AccessType::ALTER_VIEW | AccessType::DROP_DATABASE | AccessType::DROP_TABLE | AccessType::DROP_VIEW - | AccessType::TRUNCATE; - - static const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY; - static const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl; - static const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE; - static const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS; - - if (readonly_) - merged_access->revoke(write_table_access | table_and_dictionary_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY); - - if (readonly_ == 1) - { - /// Table functions are forbidden in readonly mode. - /// For example, for readonly = 2 - allowed. - merged_access->revoke(AccessType::CREATE_TEMPORARY_TABLE); - } - - if (!allow_ddl_) - merged_access->revoke(table_and_dictionary_ddl); - - if (!allow_introspection_) - merged_access->revoke(AccessType::INTROSPECTION); - - /// Anyone has access to the "system" database. - merged_access->grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE); - - if (readonly_ != 1) - { - /// User has access to temporary or external table if such table was resolved in session or query context - merged_access->grant(AccessFlags::allTableFlags() | AccessFlags::allColumnFlags(), DatabaseCatalog::TEMPORARY_DATABASE); - } - - if (readonly_ && grant_option) - { - /// No grant option in readonly mode. - merged_access->revoke(AccessType::ALL); - } - - if (trace_log && (params.readonly == readonly_) && (params.allow_ddl == allow_ddl_) && (params.allow_introspection == allow_introspection_)) - { - if (grant_option) - LOG_TRACE(trace_log, "List of all grants: {} WITH GRANT OPTION", merged_access->toString()); - else - LOG_TRACE(trace_log, "List of all grants: {}", merged_access->toString()); - - if (roles_info && !roles_info->getCurrentRolesNames().empty()) - { - LOG_TRACE(trace_log, "Current_roles: {}, enabled_roles: {}", - boost::algorithm::join(roles_info->getCurrentRolesNames(), ", "), - boost::algorithm::join(roles_info->getEnabledRolesNames(), ", ")); - } - LOG_TRACE(trace_log, "Settings: readonly={}, allow_ddl={}, allow_introspection_functions={}", readonly_, allow_ddl_, allow_introspection_); - } - - res = std::move(merged_access); - result_access[index].store(res); - return res; -} - - UserPtr ContextAccess::getUser() const { std::lock_guard lock{mutex}; @@ -520,9 +241,7 @@ std::shared_ptr ContextAccess::getFullAccess() static const std::shared_ptr res = [] { auto full_access = std::shared_ptr(new ContextAccess); - auto everything_granted = boost::make_shared(); - everything_granted->grant(AccessType::ALL); - boost::range::fill(full_access->result_access, everything_granted); + full_access->access = std::make_shared(AccessRights::getFullAccess()); full_access->enabled_quota = EnabledQuota::getUnlimitedQuota(); return full_access; }(); @@ -543,4 +262,246 @@ std::shared_ptr ContextAccess::getSettingsConstraints return enabled_settings ? enabled_settings->getConstraints() : nullptr; } + +std::shared_ptr ContextAccess::getAccess() const +{ + std::lock_guard lock{mutex}; + return access; +} + + +template +bool ContextAccess::isGrantedImpl2(const AccessFlags & flags, const Args &... args) const +{ + bool access_granted; + if constexpr (grant_option) + access_granted = getAccess()->hasGrantOption(flags, args...); + else + access_granted = getAccess()->isGranted(flags, args...); + + if (trace_log) + LOG_TRACE(trace_log, "Access {}: {}{}", (access_granted ? "granted" : "denied"), (AccessRightsElement{flags, args...}.toString()), + (grant_option ? " WITH GRANT OPTION" : "")); + + return access_granted; +} + +template +bool ContextAccess::isGrantedImpl(const AccessFlags & flags) const +{ + return isGrantedImpl2(flags); +} + +template +bool ContextAccess::isGrantedImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const +{ + return isGrantedImpl2(flags, database.empty() ? params.current_database : database, args...); +} + +template +bool ContextAccess::isGrantedImpl(const AccessRightsElement & element) const +{ + if (element.any_database) + return isGrantedImpl(element.access_flags); + else if (element.any_table) + return isGrantedImpl(element.access_flags, element.database); + else if (element.any_column) + return isGrantedImpl(element.access_flags, element.database, element.table); + else + return isGrantedImpl(element.access_flags, element.database, element.table, element.columns); +} + +template +bool ContextAccess::isGrantedImpl(const AccessRightsElements & elements) const +{ + for (const auto & element : elements) + if (!isGrantedImpl(element)) + return false; + return true; +} + +bool ContextAccess::isGranted(const AccessFlags & flags) const { return isGrantedImpl(flags); } +bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl(flags, database); } +bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl(flags, database, table); } +bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl(flags, database, table, column); } +bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { return isGrantedImpl(flags, database, table, columns); } +bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl(flags, database, table, columns); } +bool ContextAccess::isGranted(const AccessRightsElement & element) const { return isGrantedImpl(element); } +bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return isGrantedImpl(elements); } + +bool ContextAccess::hasGrantOption(const AccessFlags & flags) const { return isGrantedImpl(flags); } +bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database) const { return isGrantedImpl(flags, database); } +bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl(flags, database, table); } +bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl(flags, database, table, column); } +bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { return isGrantedImpl(flags, database, table, columns); } +bool ContextAccess::hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl(flags, database, table, columns); } +bool ContextAccess::hasGrantOption(const AccessRightsElement & element) const { return isGrantedImpl(element); } +bool ContextAccess::hasGrantOption(const AccessRightsElements & elements) const { return isGrantedImpl(elements); } + + +template +void ContextAccess::checkAccessImpl2(const AccessFlags & flags, const Args &... args) const +{ + if constexpr (grant_option) + { + if (hasGrantOption(flags, args...)) + return; + } + else + { + if (isGranted(flags, args...)) + return; + } + + auto show_error = [&](const String & msg, int error_code) + { + throw Exception(user_name + ": " + msg, error_code); + }; + + std::lock_guard lock{mutex}; + + if (!user) + { + show_error("User has been dropped", ErrorCodes::UNKNOWN_USER); + } + + if (grant_option && access->isGranted(flags, args...)) + { + show_error( + "Not enough privileges. " + "The required privileges have been granted, but without grant option. " + "To execute this query it's necessary to have the grant " + + AccessRightsElement{flags, args...}.toString() + " WITH GRANT OPTION", + ErrorCodes::ACCESS_DENIED); + } + + if (params.readonly) + { + if (!access_without_readonly) + { + Params changed_params = params; + changed_params.readonly = 0; + access_without_readonly = manager->getContextAccess(changed_params); + } + + if (access_without_readonly->isGranted(flags, args...)) + { + if (params.interface == ClientInfo::Interface::HTTP && params.http_method == ClientInfo::HTTPMethod::GET) + show_error( + "Cannot execute query in readonly mode. " + "For queries over HTTP, method GET implies readonly. You should use method POST for modifying queries", + ErrorCodes::READONLY); + else + show_error("Cannot execute query in readonly mode", ErrorCodes::READONLY); + } + } + + if (!params.allow_ddl) + { + if (!access_with_allow_ddl) + { + Params changed_params = params; + changed_params.allow_ddl = true; + access_with_allow_ddl = manager->getContextAccess(changed_params); + } + + if (access_with_allow_ddl->isGranted(flags, args...)) + { + show_error("Cannot execute query. DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED); + } + } + + if (!params.allow_introspection) + { + if (!access_with_allow_introspection) + { + Params changed_params = params; + changed_params.allow_introspection = true; + access_with_allow_introspection = manager->getContextAccess(changed_params); + } + + if (access_with_allow_introspection->isGranted(flags, args...)) + { + show_error("Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED); + } + } + + show_error( + "Not enough privileges. To execute this query it's necessary to have the grant " + + AccessRightsElement{flags, args...}.toString() + (grant_option ? " WITH GRANT OPTION" : ""), + ErrorCodes::ACCESS_DENIED); +} + +template +void ContextAccess::checkAccessImpl(const AccessFlags & flags) const +{ + checkAccessImpl2(flags); +} + +template +void ContextAccess::checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const +{ + checkAccessImpl2(flags, database.empty() ? params.current_database : database, args...); +} + +template +void ContextAccess::checkAccessImpl(const AccessRightsElement & element) const +{ + if (element.any_database) + checkAccessImpl(element.access_flags); + else if (element.any_table) + checkAccessImpl(element.access_flags, element.database); + else if (element.any_column) + checkAccessImpl(element.access_flags, element.database, element.table); + else + checkAccessImpl(element.access_flags, element.database, element.table, element.columns); +} + +template +void ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const +{ + for (const auto & element : elements) + checkAccessImpl(element); +} + +void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl(flags); } +void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl(flags, database); } +void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl(flags, database, table); } +void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl(flags, database, table, column); } +void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { checkAccessImpl(flags, database, table, columns); } +void ContextAccess::checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl(flags, database, table, columns); } +void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl(element); } +void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl(elements); } + +void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl(flags); } +void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database) const { checkAccessImpl(flags, database); } +void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const { checkAccessImpl(flags, database, table); } +void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkAccessImpl(flags, database, table, column); } +void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const { checkAccessImpl(flags, database, table, columns); } +void ContextAccess::checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkAccessImpl(flags, database, table, columns); } +void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl(element); } +void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl(elements); } + + +void ContextAccess::checkAdminOption(const UUID & role_id) const +{ + if (isGranted(AccessType::ROLE_ADMIN)) + return; + + auto info = getRolesInfo(); + if (info && info->enabled_roles_with_admin_option.count(role_id)) + return; + + if (!user) + throw Exception(user_name + ": User has been dropped", ErrorCodes::UNKNOWN_USER); + + std::optional role_name = manager->readName(role_id); + if (!role_name) + role_name = "ID {" + toString(role_id) + "}"; + throw Exception( + getUserName() + ": Not enough privileges. To execute this query it's necessary to have the grant " + backQuoteIfNeed(*role_name) + + " WITH ADMIN OPTION ", + ErrorCodes::ACCESS_DENIED); +} + } diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index 27bb29a878c..997ea585c68 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -6,7 +6,6 @@ #include #include #include -#include #include #include @@ -30,32 +29,34 @@ class IAST; using ASTPtr = std::shared_ptr; +struct ContextAccessParams +{ + std::optional user_id; + boost::container::flat_set current_roles; + bool use_default_roles = false; + UInt64 readonly = 0; + bool allow_ddl = false; + bool allow_introspection = false; + String current_database; + ClientInfo::Interface interface = ClientInfo::Interface::TCP; + ClientInfo::HTTPMethod http_method = ClientInfo::HTTPMethod::UNKNOWN; + Poco::Net::IPAddress address; + String quota_key; + + auto toTuple() const { return std::tie(user_id, current_roles, use_default_roles, readonly, allow_ddl, allow_introspection, current_database, interface, http_method, address, quota_key); } + friend bool operator ==(const ContextAccessParams & lhs, const ContextAccessParams & rhs) { return lhs.toTuple() == rhs.toTuple(); } + friend bool operator !=(const ContextAccessParams & lhs, const ContextAccessParams & rhs) { return !(lhs == rhs); } + friend bool operator <(const ContextAccessParams & lhs, const ContextAccessParams & rhs) { return lhs.toTuple() < rhs.toTuple(); } + friend bool operator >(const ContextAccessParams & lhs, const ContextAccessParams & rhs) { return rhs < lhs; } + friend bool operator <=(const ContextAccessParams & lhs, const ContextAccessParams & rhs) { return !(rhs < lhs); } + friend bool operator >=(const ContextAccessParams & lhs, const ContextAccessParams & rhs) { return !(lhs < rhs); } +}; + + class ContextAccess { public: - struct Params - { - std::optional user_id; - boost::container::flat_set current_roles; - bool use_default_roles = false; - UInt64 readonly = 0; - bool allow_ddl = false; - bool allow_introspection = false; - String current_database; - ClientInfo::Interface interface = ClientInfo::Interface::TCP; - ClientInfo::HTTPMethod http_method = ClientInfo::HTTPMethod::UNKNOWN; - Poco::Net::IPAddress address; - String quota_key; - - auto toTuple() const { return std::tie(user_id, current_roles, use_default_roles, readonly, allow_ddl, allow_introspection, current_database, interface, http_method, address, quota_key); } - friend bool operator ==(const Params & lhs, const Params & rhs) { return lhs.toTuple() == rhs.toTuple(); } - friend bool operator !=(const Params & lhs, const Params & rhs) { return !(lhs == rhs); } - friend bool operator <(const Params & lhs, const Params & rhs) { return lhs.toTuple() < rhs.toTuple(); } - friend bool operator >(const Params & lhs, const Params & rhs) { return rhs < lhs; } - friend bool operator <=(const Params & lhs, const Params & rhs) { return !(rhs < lhs); } - friend bool operator >=(const Params & lhs, const Params & rhs) { return !(lhs < rhs); } - }; - + using Params = ContextAccessParams; const Params & getParams() const { return params; } /// Returns the current user. The function can return nullptr. @@ -90,16 +91,8 @@ public: /// The function returns nullptr if there are no constraints. std::shared_ptr getSettingsConstraints() const; - /// Checks if a specified access is granted, and throws an exception if not. - /// Empty database means the current database. - void checkAccess(const AccessFlags & flags) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; - void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; - void checkAccess(const AccessRightsElement & element) const; - void checkAccess(const AccessRightsElements & elements) const; + /// Returns the current access rights. + std::shared_ptr getAccess() const; /// Checks if a specified access is granted. bool isGranted(const AccessFlags & flags) const; @@ -111,17 +104,26 @@ public: bool isGranted(const AccessRightsElement & element) const; bool isGranted(const AccessRightsElements & elements) const; - /// Checks if a specified access is granted, and logs a warning if not. - bool isGranted(Poco::Logger * log_, const AccessFlags & flags) const; - bool isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database) const; - bool isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; - bool isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; - bool isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; - bool isGranted(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; - bool isGranted(Poco::Logger * log_, const AccessRightsElement & element) const; - bool isGranted(Poco::Logger * log_, const AccessRightsElements & elements) const; + bool hasGrantOption(const AccessFlags & flags) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; + bool hasGrantOption(const AccessRightsElement & element) const; + bool hasGrantOption(const AccessRightsElements & elements) const; + + /// Checks if a specified access is granted, and throws an exception if not. + /// Empty database means the current database. + void checkAccess(const AccessFlags & flags) const; + void checkAccess(const AccessFlags & flags, const std::string_view & database) const; + void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; + void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; + void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; + void checkAccess(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; + void checkAccess(const AccessRightsElement & element) const; + void checkAccess(const AccessRightsElements & elements) const; - /// Checks if a specified access is granted with grant option, and throws an exception if not. void checkGrantOption(const AccessFlags & flags) const; void checkGrantOption(const AccessFlags & flags, const std::string_view & database) const; void checkGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; @@ -146,24 +148,37 @@ private: void setUser(const UserPtr & user_) const; void setRolesInfo(const std::shared_ptr & roles_info_) const; void setSettingsAndConstraints() const; + void setFinalAccess() const; - template - bool checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags) const; + template + bool isGrantedImpl(const AccessFlags & flags) const; - template - bool checkAccessImpl(Poco::Logger * log_, const AccessFlags & flags, const std::string_view & database, const Args &... args) const; + template + bool isGrantedImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const; - template - bool checkAccessImpl(Poco::Logger * log_, const AccessRightsElement & element) const; + template + bool isGrantedImpl(const AccessRightsElement & element) const; - template - bool checkAccessImpl(Poco::Logger * log_, const AccessRightsElements & elements) const; + template + bool isGrantedImpl(const AccessRightsElements & elements) const; - template - bool calculateResultAccessAndCheck(Poco::Logger * log_, const AccessFlags & flags, const Args &... args) const; + template + bool isGrantedImpl2(const AccessFlags & flags, const Args &... args) const; - boost::shared_ptr calculateResultAccess(bool grant_option) const; - boost::shared_ptr calculateResultAccess(bool grant_option, UInt64 readonly_, bool allow_ddl_, bool allow_introspection_) const; + template + void checkAccessImpl(const AccessFlags & flags) const; + + template + void checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const; + + template + void checkAccessImpl(const AccessRightsElement & element) const; + + template + void checkAccessImpl(const AccessRightsElements & elements) const; + + template + void checkAccessImpl2(const AccessFlags & flags, const Args &... args) const; const AccessControlManager * manager = nullptr; const Params params; @@ -174,10 +189,13 @@ private: mutable std::shared_ptr enabled_roles; mutable ext::scope_guard subscription_for_roles_changes; mutable std::shared_ptr roles_info; - mutable boost::atomic_shared_ptr result_access[7]; + mutable std::shared_ptr access; mutable std::shared_ptr enabled_row_policies; mutable std::shared_ptr enabled_quota; mutable std::shared_ptr enabled_settings; + mutable std::shared_ptr access_without_readonly; + mutable std::shared_ptr access_with_allow_ddl; + mutable std::shared_ptr access_with_allow_introspection; mutable std::mutex mutex; }; diff --git a/src/Access/EnabledRolesInfo.cpp b/src/Access/EnabledRolesInfo.cpp index 01b90d6fa1e..8069da467ad 100644 --- a/src/Access/EnabledRolesInfo.cpp +++ b/src/Access/EnabledRolesInfo.cpp @@ -28,8 +28,7 @@ bool operator==(const EnabledRolesInfo & lhs, const EnabledRolesInfo & rhs) { return (lhs.current_roles == rhs.current_roles) && (lhs.enabled_roles == rhs.enabled_roles) && (lhs.enabled_roles_with_admin_option == rhs.enabled_roles_with_admin_option) && (lhs.names_of_roles == rhs.names_of_roles) - && (lhs.access == rhs.access) && (lhs.access_with_grant_option == rhs.access_with_grant_option) - && (lhs.settings_from_enabled_roles == rhs.settings_from_enabled_roles); + && (lhs.access == rhs.access) && (lhs.settings_from_enabled_roles == rhs.settings_from_enabled_roles); } } diff --git a/src/Access/EnabledRolesInfo.h b/src/Access/EnabledRolesInfo.h index 45e1bfd9057..f06b7478daf 100644 --- a/src/Access/EnabledRolesInfo.h +++ b/src/Access/EnabledRolesInfo.h @@ -18,7 +18,6 @@ struct EnabledRolesInfo boost::container::flat_set enabled_roles_with_admin_option; std::unordered_map names_of_roles; AccessRights access; - AccessRights access_with_grant_option; SettingsProfileElements settings_from_enabled_roles; Strings getCurrentRolesNames() const; diff --git a/src/Access/GrantedAccess.cpp b/src/Access/GrantedAccess.cpp deleted file mode 100644 index 2af1e0b44ec..00000000000 --- a/src/Access/GrantedAccess.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include - - -namespace DB -{ - -GrantedAccess::GrantsAndPartialRevokes GrantedAccess::getGrantsAndPartialRevokes() const -{ - GrantsAndPartialRevokes res; - res.grants_with_grant_option = access_with_grant_option.getGrants(); - AccessRights access_without_gg = access; - access_without_gg.revoke(res.grants_with_grant_option); - auto gr = access_without_gg.getGrantsAndPartialRevokes(); - res.grants = std::move(gr.grants); - res.revokes = std::move(gr.revokes); - AccessRights access_with_grant_options_without_r = access_with_grant_option; - access_with_grant_options_without_r.grant(res.revokes); - res.revokes_grant_option = access_with_grant_options_without_r.getPartialRevokes(); - return res; -} - -} diff --git a/src/Access/GrantedAccess.h b/src/Access/GrantedAccess.h deleted file mode 100644 index b8f6bdfe8fb..00000000000 --- a/src/Access/GrantedAccess.h +++ /dev/null @@ -1,55 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ -/// Access rights as they are granted to a role or user. -/// Stores both the access rights themselves and the access rights with grant option. -struct GrantedAccess -{ - AccessRights access; - AccessRights access_with_grant_option; - - template - void grant(const Args &... args) - { - access.grant(args...); - } - - template - void grantWithGrantOption(const Args &... args) - { - access.grant(args...); - access_with_grant_option.grant(args...); - } - - template - void revoke(const Args &... args) - { - access.revoke(args...); - access_with_grant_option.revoke(args...); - } - - template - void revokeGrantOption(const Args &... args) - { - access_with_grant_option.revoke(args...); - } - - struct GrantsAndPartialRevokes - { - AccessRightsElements grants; - AccessRightsElements revokes; - AccessRightsElements grants_with_grant_option; - AccessRightsElements revokes_grant_option; - }; - - /// Retrieves the information about grants and partial revokes. - GrantsAndPartialRevokes getGrantsAndPartialRevokes() const; - - friend bool operator ==(const GrantedAccess & left, const GrantedAccess & right) { return (left.access == right.access) && (left.access_with_grant_option == right.access_with_grant_option); } - friend bool operator !=(const GrantedAccess & left, const GrantedAccess & right) { return !(left == right); } -}; -} diff --git a/src/Access/Role.h b/src/Access/Role.h index 9acb97bdfbd..131bbd69195 100644 --- a/src/Access/Role.h +++ b/src/Access/Role.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include @@ -11,7 +11,7 @@ namespace DB struct Role : public IAccessEntity { - GrantedAccess access; + AccessRights access; GrantedRoles granted_roles; SettingsProfileElements settings; diff --git a/src/Access/RoleCache.cpp b/src/Access/RoleCache.cpp index ca8065145f3..a0468958d42 100644 --- a/src/Access/RoleCache.cpp +++ b/src/Access/RoleCache.cpp @@ -43,8 +43,7 @@ namespace roles_info.enabled_roles_with_admin_option.emplace(role_id); roles_info.names_of_roles[role_id] = role->getName(); - roles_info.access.merge(role->access.access); - roles_info.access_with_grant_option.merge(role->access.access_with_grant_option); + roles_info.access.merge(role->access); roles_info.settings_from_enabled_roles.merge(role->settings); for (const auto & granted_role : role->granted_roles.roles) diff --git a/src/Access/User.h b/src/Access/User.h index 4852fce375d..13f1e532015 100644 --- a/src/Access/User.h +++ b/src/Access/User.h @@ -1,9 +1,9 @@ #pragma once #include +#include #include #include -#include #include #include #include @@ -17,7 +17,7 @@ struct User : public IAccessEntity { Authentication authentication; AllowedClientHosts allowed_client_hosts = AllowedClientHosts::AnyHostTag{}; - GrantedAccess access; + AccessRights access; GrantedRoles granted_roles; RolesOrUsersSet default_roles = RolesOrUsersSet::AllTag{}; SettingsProfileElements settings; diff --git a/src/Access/ya.make b/src/Access/ya.make index bdd62ae2b7b..77c94b87dfa 100644 --- a/src/Access/ya.make +++ b/src/Access/ya.make @@ -17,7 +17,6 @@ SRCS( EnabledRolesInfo.cpp EnabledRowPolicies.cpp EnabledSettings.cpp - GrantedAccess.cpp GrantedRoles.cpp IAccessEntity.cpp IAccessStorage.cpp diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 28436f192b0..2278c0e452f 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -1278,7 +1279,7 @@ private: }; -BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & context, AccessRightsElements && query_required_access) +BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & context, AccessRightsElements && query_requires_access, bool query_requires_grant_option) { /// Remove FORMAT and INTO OUTFILE if exists ASTPtr query_ptr = query_ptr_->clone(); @@ -1323,10 +1324,10 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & cont /// the local current database or a shard's default database. bool need_replace_current_database = (std::find_if( - query_required_access.begin(), - query_required_access.end(), + query_requires_access.begin(), + query_requires_access.end(), [](const AccessRightsElement & elem) { return elem.isEmptyDatabase(); }) - != query_required_access.end()); + != query_requires_access.end()); if (need_replace_current_database) { @@ -1355,29 +1356,31 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & cont AddDefaultDatabaseVisitor visitor(current_database); visitor.visitDDL(query_ptr); - query_required_access.replaceEmptyDatabase(current_database); + query_requires_access.replaceEmptyDatabase(current_database); } else { - size_t old_num_elements = query_required_access.size(); - for (size_t i = 0; i != old_num_elements; ++i) + for (size_t i = 0; i != query_requires_access.size();) { - auto & element = query_required_access[i]; + auto & element = query_requires_access[i]; if (element.isEmptyDatabase()) { - element.setDatabase(shard_default_databases[0]); - for (size_t j = 1; j != shard_default_databases.size(); ++j) - { - query_required_access.push_back(element); - query_required_access.back().setDatabase(shard_default_databases[j]); - } + query_requires_access.insert(query_requires_access.begin() + i + 1, shard_default_databases.size() - 1, element); + for (size_t j = 0; j != shard_default_databases.size(); ++j) + query_requires_access[i + j].replaceEmptyDatabase(shard_default_databases[j]); + i += shard_default_databases.size(); } + else + ++i; } } } /// Check access rights, assume that all servers have the same users config - context.checkAccess(query_required_access); + if (query_requires_grant_option) + context.getAccess()->checkGrantOption(query_requires_access); + else + context.checkAccess(query_requires_access); DDLLogEntry entry; entry.hosts = std::move(hosts); @@ -1394,6 +1397,10 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & cont return io; } +BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, const AccessRightsElements & query_requires_access, bool query_requires_grant_option) +{ + return executeDDLQueryOnCluster(query_ptr, context, AccessRightsElements{query_requires_access}, query_requires_grant_option); +} BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & context) { diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index d764eab626f..544fb3da27d 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -29,8 +29,9 @@ struct DDLTask; /// Pushes distributed DDL query to the queue -BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, AccessRightsElements && query_required_access); BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context); +BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, const AccessRightsElements & query_requires_access, bool query_requires_grant_option = false); +BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, AccessRightsElements && query_requires_access, bool query_requires_grant_option = false); class DDLWorker diff --git a/src/Interpreters/InterpreterGrantQuery.cpp b/src/Interpreters/InterpreterGrantQuery.cpp index 8981c06f962..b7c62197059 100644 --- a/src/Interpreters/InterpreterGrantQuery.cpp +++ b/src/Interpreters/InterpreterGrantQuery.cpp @@ -16,7 +16,7 @@ namespace DB namespace { template - void updateFromQueryImpl(T & grantee, const ASTGrantQuery & query, const std::vector & roles_from_query, const String & current_database) + void updateFromQueryImpl(T & grantee, const ASTGrantQuery & query, const std::vector & roles_from_query) { using Kind = ASTGrantQuery::Kind; if (!query.access_rights_elements.empty()) @@ -24,16 +24,16 @@ namespace if (query.kind == Kind::GRANT) { if (query.grant_option) - grantee.access.grantWithGrantOption(query.access_rights_elements, current_database); + grantee.access.grantWithGrantOption(query.access_rights_elements); else - grantee.access.grant(query.access_rights_elements, current_database); + grantee.access.grant(query.access_rights_elements); } else { if (query.grant_option) - grantee.access.revokeGrantOption(query.access_rights_elements, current_database); + grantee.access.revokeGrantOption(query.access_rights_elements); else - grantee.access.revoke(query.access_rights_elements, current_database); + grantee.access.revoke(query.access_rights_elements); } } @@ -67,9 +67,9 @@ namespace BlockIO InterpreterGrantQuery::execute() { auto & query = query_ptr->as(); - auto & access_control = context.getAccessControlManager(); + query.replaceCurrentUserTagWithName(context.getUserName()); auto access = context.getAccess(); - access->checkGrantOption(query.access_rights_elements); + auto & access_control = context.getAccessControlManager(); std::vector roles_from_query; if (query.roles) @@ -80,25 +80,24 @@ BlockIO InterpreterGrantQuery::execute() } if (!query.cluster.empty()) - { - query.replaceCurrentUserTagWithName(context.getUserName()); - return executeDDLQueryOnCluster(query_ptr, context); - } + return executeDDLQueryOnCluster(query_ptr, context, query.access_rights_elements, true); + + query.replaceEmptyDatabaseWithCurrent(context.getCurrentDatabase()); + access->checkGrantOption(query.access_rights_elements); std::vector to_roles = RolesOrUsersSet{*query.to_roles, access_control, context.getUserID()}.getMatchingIDs(access_control); - String current_database = context.getCurrentDatabase(); auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr { auto clone = entity->clone(); if (auto user = typeid_cast>(clone)) { - updateFromQueryImpl(*user, query, roles_from_query, current_database); + updateFromQueryImpl(*user, query, roles_from_query); return user; } else if (auto role = typeid_cast>(clone)) { - updateFromQueryImpl(*role, query, roles_from_query, current_database); + updateFromQueryImpl(*role, query, roles_from_query); return role; } else @@ -116,7 +115,7 @@ void InterpreterGrantQuery::updateUserFromQuery(User & user, const ASTGrantQuery std::vector roles_from_query; if (query.roles) roles_from_query = RolesOrUsersSet{*query.roles}.getMatchingIDs(); - updateFromQueryImpl(user, query, roles_from_query, {}); + updateFromQueryImpl(user, query, roles_from_query); } @@ -125,7 +124,7 @@ void InterpreterGrantQuery::updateRoleFromQuery(Role & role, const ASTGrantQuery std::vector roles_from_query; if (query.roles) roles_from_query = RolesOrUsersSet{*query.roles}.getMatchingIDs(); - updateFromQueryImpl(role, query, roles_from_query, {}); + updateFromQueryImpl(role, query, roles_from_query); } } diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 82c134aeba6..80710600db6 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -84,19 +84,20 @@ static QueryDescriptors extractQueriesExceptMeAndCheckAccess(const Block & proce const ColumnString & user_col = typeid_cast(*processes_block.getByName("user").column); const ClientInfo & my_client = context.getProcessListElement()->getClientInfo(); - std::optional can_kill_query_started_by_another_user_cached; - auto can_kill_query_started_by_another_user = [&]() -> bool + bool access_denied = false; + std::optional is_kill_query_granted_value; + auto is_kill_query_granted = [&]() -> bool { - if (!can_kill_query_started_by_another_user_cached) + if (!is_kill_query_granted_value) { - can_kill_query_started_by_another_user_cached - = context.getAccess()->isGranted(&Poco::Logger::get("InterpreterKillQueryQuery"), AccessType::KILL_QUERY); + is_kill_query_granted_value = context.getAccess()->isGranted(AccessType::KILL_QUERY); + if (!*is_kill_query_granted_value) + access_denied = true; } - return *can_kill_query_started_by_another_user_cached; + return *is_kill_query_granted_value; }; String query_user; - bool access_denied = false; for (size_t i = 0; i < num_processes; ++i) { @@ -107,11 +108,8 @@ static QueryDescriptors extractQueriesExceptMeAndCheckAccess(const Block & proce auto query_id = query_id_col.getDataAt(i).toString(); query_user = user_col.getDataAt(i).toString(); - if ((my_client.current_user != query_user) && !can_kill_query_started_by_another_user()) - { - access_denied = true; + if ((my_client.current_user != query_user) && !is_kill_query_granted()) continue; - } res.emplace_back(std::move(query_id), query_user, i, false); } @@ -269,7 +267,7 @@ BlockIO InterpreterKillQueryQuery::execute() ParserAlterCommand parser; auto command_ast = parseQuery(parser, command_col.getDataAt(i).toString(), 0, context.getSettingsRef().max_parser_depth); required_access_rights = InterpreterAlterQuery::getRequiredAccessForCommand(command_ast->as(), table_id.database_name, table_id.table_name); - if (!access->isGranted(&Poco::Logger::get("InterpreterKillQueryQuery"), required_access_rights)) + if (!access->isGranted(required_access_rights)) { access_denied = true; continue; diff --git a/src/Interpreters/InterpreterShowGrantsQuery.cpp b/src/Interpreters/InterpreterShowGrantsQuery.cpp index ebb0d871c8b..45e065dcfd9 100644 --- a/src/Interpreters/InterpreterShowGrantsQuery.cpp +++ b/src/Interpreters/InterpreterShowGrantsQuery.cpp @@ -35,44 +35,33 @@ namespace std::shared_ptr to_roles = std::make_shared(); to_roles->names.push_back(grantee.getName()); - auto grants_and_partial_revokes = grantee.access.getGrantsAndPartialRevokes(); + std::shared_ptr current_query = nullptr; - for (bool grant_option : {false, true}) + auto elements = grantee.access.getElements(); + for (const auto & element : elements) { - using Kind = ASTGrantQuery::Kind; - for (Kind kind : {Kind::GRANT, Kind::REVOKE}) + if (current_query) { - AccessRightsElements * elements = nullptr; - if (grant_option) - elements = (kind == Kind::GRANT) ? &grants_and_partial_revokes.grants_with_grant_option : &grants_and_partial_revokes.revokes_grant_option; - else - elements = (kind == Kind::GRANT) ? &grants_and_partial_revokes.grants : &grants_and_partial_revokes.revokes; - elements->normalize(); - - std::shared_ptr grant_query = nullptr; - for (size_t i = 0; i != elements->size(); ++i) - { - const auto & element = (*elements)[i]; - bool prev_element_on_same_db_and_table = false; - if (grant_query) - { - const auto & prev_element = grant_query->access_rights_elements.back(); - if ((element.database == prev_element.database) && (element.any_database == prev_element.any_database) - && (element.table == prev_element.table) && (element.any_table == prev_element.any_table)) - prev_element_on_same_db_and_table = true; - } - if (!prev_element_on_same_db_and_table) - { - grant_query = std::make_shared(); - grant_query->kind = kind; - grant_query->attach = attach_mode; - grant_query->grant_option = grant_option; - grant_query->to_roles = to_roles; - res.push_back(grant_query); - } - grant_query->access_rights_elements.emplace_back(std::move(element)); - } + const auto & prev_element = current_query->access_rights_elements.back(); + bool continue_using_current_query = (element.database == prev_element.database) + && (element.any_database == prev_element.any_database) && (element.table == prev_element.table) + && (element.any_table == prev_element.any_table) && (element.grant_option == current_query->grant_option) + && (element.kind == current_query->kind); + if (!continue_using_current_query) + current_query = nullptr; } + + if (!current_query) + { + current_query = std::make_shared(); + current_query->kind = element.kind; + current_query->attach = attach_mode; + current_query->grant_option = element.grant_option; + current_query->to_roles = to_roles; + res.push_back(current_query); + } + + current_query->access_rights_elements.emplace_back(std::move(element)); } auto grants_roles = grantee.granted_roles.getGrants(); diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 9b1712ac407..7c80b681114 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -152,8 +152,16 @@ void InterpreterSystemQuery::startStopAction(StorageActionBlockType action_type, if (!table) continue; - if (!access->isGranted(log, getRequiredAccessType(action_type), elem.first, iterator->name())) + if (!access->isGranted(getRequiredAccessType(action_type), elem.first, iterator->name())) + { + LOG_INFO( + log, + "Access {} denied, skipping {}.{}", + toString(getRequiredAccessType(action_type)), + elem.first, + iterator->name()); continue; + } if (start) manager->remove(table, action_type); diff --git a/src/Interpreters/tests/users.cpp b/src/Interpreters/tests/users.cpp index 5c7d66ed7ed..acd0cfd0519 100644 --- a/src/Interpreters/tests/users.cpp +++ b/src/Interpreters/tests/users.cpp @@ -218,7 +218,7 @@ void runOneTest(const TestDescriptor & test_descriptor) try { - res = acl_manager.read(entry.user_name)->access.access.isGranted(DB::AccessType::ALL, entry.database_name); + res = acl_manager.read(entry.user_name)->access.isGranted(DB::AccessType::ALL, entry.database_name); } catch (const Poco::Exception &) { diff --git a/src/Parsers/ASTGrantQuery.cpp b/src/Parsers/ASTGrantQuery.cpp index cf1943477b2..ae9649cdddc 100644 --- a/src/Parsers/ASTGrantQuery.cpp +++ b/src/Parsers/ASTGrantQuery.cpp @@ -133,6 +133,12 @@ void ASTGrantQuery::formatImpl(const FormatSettings & settings, FormatState &, F } +void ASTGrantQuery::replaceEmptyDatabaseWithCurrent(const String & current_database) +{ + access_rights_elements.replaceEmptyDatabase(current_database); +} + + void ASTGrantQuery::replaceCurrentUserTagWithName(const String & current_user_name) const { if (to_roles) diff --git a/src/Parsers/ASTGrantQuery.h b/src/Parsers/ASTGrantQuery.h index 9a11f5dc509..c36e42689a5 100644 --- a/src/Parsers/ASTGrantQuery.h +++ b/src/Parsers/ASTGrantQuery.h @@ -19,11 +19,7 @@ class ASTRolesOrUsersSet; class ASTGrantQuery : public IAST, public ASTQueryWithOnCluster { public: - enum class Kind - { - GRANT, - REVOKE, - }; + using Kind = AccessRightsElementWithOptions::Kind; Kind kind = Kind::GRANT; bool attach = false; AccessRightsElements access_rights_elements; @@ -35,6 +31,7 @@ public: String getID(char) const override; ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + void replaceEmptyDatabaseWithCurrent(const String & current_database); void replaceCurrentUserTagWithName(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } }; diff --git a/src/Storages/System/StorageSystemGrants.cpp b/src/Storages/System/StorageSystemGrants.cpp index a663e3307fe..360256c1f45 100644 --- a/src/Storages/System/StorageSystemGrants.cpp +++ b/src/Storages/System/StorageSystemGrants.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -17,7 +18,7 @@ namespace DB { using EntityType = IAccessEntity::Type; - +using Kind = AccessRightsElementWithOptions::Kind; NamesAndTypesList StorageSystemGrants::getNamesAndTypes() { @@ -63,7 +64,7 @@ void StorageSystemGrants::fillData(MutableColumns & res_columns, const Context & const String * database, const String * table, const String * column, - bool is_partial_revoke, + Kind kind, bool grant_option) { if (grantee_type == EntityType::USER) @@ -118,15 +119,13 @@ void StorageSystemGrants::fillData(MutableColumns & res_columns, const Context & column_column_null_map.push_back(true); } - column_is_partial_revoke.push_back(is_partial_revoke); + column_is_partial_revoke.push_back(kind == Kind::REVOKE); column_grant_option.push_back(grant_option); }; auto add_rows = [&](const String & grantee_name, IAccessEntity::Type grantee_type, - const AccessRightsElements & elements, - bool is_partial_revoke, - bool grant_option) + const AccessRightsElementsWithOptions & elements) { for (const auto & element : elements) { @@ -140,13 +139,13 @@ void StorageSystemGrants::fillData(MutableColumns & res_columns, const Context & if (element.any_column) { for (const auto & access_type : access_types) - add_row(grantee_name, grantee_type, access_type, database, table, nullptr, is_partial_revoke, grant_option); + add_row(grantee_name, grantee_type, access_type, database, table, nullptr, element.kind, element.grant_option); } else { for (const auto & access_type : access_types) for (const auto & column : element.columns) - add_row(grantee_name, grantee_type, access_type, database, table, &column, is_partial_revoke, grant_option); + add_row(grantee_name, grantee_type, access_type, database, table, &column, element.kind, element.grant_option); } } }; @@ -157,7 +156,7 @@ void StorageSystemGrants::fillData(MutableColumns & res_columns, const Context & if (!entity) continue; - const GrantedAccess * access = nullptr; + const AccessRights * access = nullptr; if (auto role = typeid_cast(entity)) access = &role->access; else if (auto user = typeid_cast(entity)) @@ -167,13 +166,8 @@ void StorageSystemGrants::fillData(MutableColumns & res_columns, const Context & const String & grantee_name = entity->getName(); const auto grantee_type = entity->getType(); - auto grants_and_revokes = access->access.getGrantsAndPartialRevokes(); - auto grants_and_revokes_with_grant_option = access->access_with_grant_option.getGrantsAndPartialRevokes(); - - add_rows(grantee_name, grantee_type, grants_and_revokes.grants, /* is_partial_revoke = */ false, /* grant_option = */ false); - add_rows(grantee_name, grantee_type, grants_and_revokes.revokes, /* is_partial_revoke = */ true, /* grant_option = */ false); - add_rows(grantee_name, grantee_type, grants_and_revokes_with_grant_option.grants, /* is_partial_revoke = */ false, /* grant_option = */ true); - add_rows(grantee_name, grantee_type, grants_and_revokes_with_grant_option.revokes, /* is_partial_revoke = */ true, /* grant_option = */ true); + auto elements = access->getElements(); + add_rows(grantee_name, grantee_type, elements); } } diff --git a/tests/integration/test_create_user_and_login/test.py b/tests/integration/test_create_user_and_login/test.py index ae75f69d28a..e1bc99ca75b 100644 --- a/tests/integration/test_create_user_and_login/test.py +++ b/tests/integration/test_create_user_and_login/test.py @@ -100,7 +100,6 @@ def test_introspection(): assert instance.query("SELECT * from system.grants WHERE user_name IN ('A', 'B') ORDER BY user_name, access_type, grant_option") ==\ TSV([[ "A", "\N", "SELECT", "test", "table", "\N", 0, 0 ], - [ "B", "\N", "CREATE", "\N", "\N", "\N", 0, 0 ], [ "B", "\N", "CREATE", "\N", "\N", "\N", 0, 1 ]]) diff --git a/tests/integration/test_role/test.py b/tests/integration/test_role/test.py index e668b461389..5fb521fc1ff 100644 --- a/tests/integration/test_role/test.py +++ b/tests/integration/test_role/test.py @@ -140,7 +140,6 @@ def test_introspection(): assert instance.query("SELECT * from system.grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, access_type, grant_option") ==\ TSV([[ "A", "\N", "SELECT", "test", "table", "\N", 0, 0 ], - [ "B", "\N", "CREATE", "\N", "\N", "\N", 0, 0 ], [ "B", "\N", "CREATE", "\N", "\N", "\N", 0, 1 ], [ "\N", "R2", "SELECT", "test", "table", "\N", 0, 0 ], [ "\N", "R2", "SELECT", "test", "table", "x", 1, 0 ]]) diff --git a/tests/queries/0_stateless/01073_grant_and_revoke.reference b/tests/queries/0_stateless/01073_grant_and_revoke.reference index 134256c8113..a19caf19533 100644 --- a/tests/queries/0_stateless/01073_grant_and_revoke.reference +++ b/tests/queries/0_stateless/01073_grant_and_revoke.reference @@ -1,11 +1,11 @@ CREATE USER test_user_01073 A B +GRANT INSERT, ALTER DELETE ON *.* TO test_user_01073 GRANT SELECT ON db1.* TO test_user_01073 GRANT SELECT ON db2.table TO test_user_01073 GRANT SELECT(col1) ON db3.table TO test_user_01073 GRANT SELECT(col1, col2) ON db4.table TO test_user_01073 -GRANT INSERT, ALTER DELETE ON *.* TO test_user_01073 C -GRANT SELECT(col1) ON db4.table TO test_user_01073 GRANT ALTER DELETE ON *.* TO test_user_01073 +GRANT SELECT(col1) ON db4.table TO test_user_01073 diff --git a/tests/queries/0_stateless/01074_partial_revokes.reference b/tests/queries/0_stateless/01074_partial_revokes.reference index 19a70679143..43e44f3c941 100644 --- a/tests/queries/0_stateless/01074_partial_revokes.reference +++ b/tests/queries/0_stateless/01074_partial_revokes.reference @@ -1,2 +1,61 @@ +--simple 1 GRANT SELECT ON *.* TO test_user_01074 REVOKE SELECT ON db.* FROM test_user_01074 +--cleanup +--simple 2 +GRANT SELECT ON db.* TO test_user_01074 +REVOKE SELECT ON db.table FROM test_user_01074 +--cleanup +--simple 3 +GRANT SELECT ON db.table TO test_user_01074 +REVOKE SELECT(col1) ON db.table FROM test_user_01074 +--cleanup +--complex 1 +GRANT SELECT ON *.* TO test_user_01074 +REVOKE SELECT(col1, col2) ON db.table FROM test_user_01074 +--cleanup +--complex 2 +GRANT SELECT ON *.* TO test_user_01074 +REVOKE SELECT ON db.* FROM test_user_01074 +GRANT SELECT ON db.table TO test_user_01074 +REVOKE SELECT(col1) ON db.table FROM test_user_01074 +┏━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ +┃ user_name  ┃ role_name ┃ access_type ┃ database ┃ table ┃ column ┃ is_partial_revoke ┃ grant_option ┃ +┡━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ +│ test_user_01074 │ ᴺᵁᴸᴸ │ SELECT │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ 0 │ +├─────────────────┼───────────┼─────────────┼──────────┼───────┼────────┼───────────────────┼──────────────┤ +│ test_user_01074 │ ᴺᵁᴸᴸ │ SELECT │ db │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 1 │ 0 │ +├─────────────────┼───────────┼─────────────┼──────────┼───────┼────────┼───────────────────┼──────────────┤ +│ test_user_01074 │ ᴺᵁᴸᴸ │ SELECT │ db │ table │ ᴺᵁᴸᴸ │ 0 │ 0 │ +├─────────────────┼───────────┼─────────────┼──────────┼───────┼────────┼───────────────────┼──────────────┤ +│ test_user_01074 │ ᴺᵁᴸᴸ │ SELECT │ db │ table │ col1 │ 1 │ 0 │ +└─────────────────┴───────────┴─────────────┴──────────┴───────┴────────┴───────────────────┴──────────────┘ +--cleanup +--revoke 1 +GRANT SELECT ON *.* TO test_user_01074 +REVOKE SELECT ON db.* FROM test_user_01074 +--cleanup +--revoke 2 +GRANT SELECT ON *.* TO test_user_01074 +--cleanup +--grant option 1 +GRANT SELECT ON *.* TO test_user_01074 WITH GRANT OPTION +REVOKE GRANT OPTION FOR SELECT(col1) ON db.table FROM test_user_01074 +┏━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ +┃ user_name  ┃ role_name ┃ access_type ┃ database ┃ table ┃ column ┃ is_partial_revoke ┃ grant_option ┃ +┡━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ +│ test_user_01074 │ ᴺᵁᴸᴸ │ SELECT │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ 0 │ 1 │ +├─────────────────┼───────────┼─────────────┼──────────┼───────┼────────┼───────────────────┼──────────────┤ +│ test_user_01074 │ ᴺᵁᴸᴸ │ SELECT │ db │ table │ col1 │ 1 │ 1 │ +└─────────────────┴───────────┴─────────────┴──────────┴───────┴────────┴───────────────────┴──────────────┘ +--cleanup +--grant option 2 +GRANT SELECT ON *.* TO test_user_01074 WITH GRANT OPTION +REVOKE SELECT(col1) ON db.table FROM test_user_01074 +--cleanup +--grant option 3 +GRANT SELECT ON *.* TO test_user_01074 +--cleanup +--grant option 4 +GRANT SELECT ON *.* TO test_user_01074 +GRANT SELECT ON db.* TO test_user_01074 WITH GRANT OPTION diff --git a/tests/queries/0_stateless/01074_partial_revokes.sql b/tests/queries/0_stateless/01074_partial_revokes.sql index 4406341cc4f..8c92b9511c7 100644 --- a/tests/queries/0_stateless/01074_partial_revokes.sql +++ b/tests/queries/0_stateless/01074_partial_revokes.sql @@ -1,8 +1,106 @@ DROP USER IF EXISTS test_user_01074; CREATE USER test_user_01074; +SELECT '--simple 1'; GRANT SELECT ON *.* TO test_user_01074; REVOKE SELECT ON db.* FROM test_user_01074; SHOW GRANTS FOR test_user_01074; +SELECT '--cleanup'; +REVOKE SELECT ON *.* FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--simple 2'; +GRANT SELECT ON db.* TO test_user_01074; +REVOKE SELECT ON db.table FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--cleanup'; +REVOKE SELECT ON *.* FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--simple 3'; +GRANT SELECT ON db.table TO test_user_01074; +REVOKE SELECT(col1) ON db.table FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--cleanup'; +REVOKE SELECT ON *.* FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--complex 1'; +GRANT SELECT ON *.* TO test_user_01074; +REVOKE SELECT(col1, col2) ON db.table FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--cleanup'; +REVOKE SELECT ON *.* FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--complex 2'; +GRANT SELECT ON *.* TO test_user_01074; +REVOKE SELECT ON db.* FROM test_user_01074; +GRANT SELECT ON db.table TO test_user_01074; +REVOKE SELECT(col1) ON db.table FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; +SELECT * FROM system.grants WHERE user_name = 'test_user_01074' format Pretty; + +SELECT '--cleanup'; +REVOKE SELECT ON *.* FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--revoke 1'; +GRANT SELECT ON *.* TO test_user_01074; +REVOKE SELECT ON db.table FROM test_user_01074; +REVOKE SELECT ON db.* FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--cleanup'; +REVOKE SELECT ON *.* FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--revoke 2'; +GRANT SELECT ON *.* TO test_user_01074; +REVOKE SELECT ON db.table FROM test_user_01074; +GRANT SELECT ON db.* TO test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--cleanup'; +REVOKE SELECT ON *.* FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--grant option 1'; +GRANT SELECT ON *.* TO test_user_01074 WITH GRANT OPTION; +REVOKE GRANT OPTION FOR SELECT(col1) ON db.table FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; +SELECT * FROM system.grants WHERE user_name = 'test_user_01074' format Pretty; + +SELECT '--cleanup'; +REVOKE SELECT ON *.* FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--grant option 2'; +GRANT SELECT ON *.* TO test_user_01074 WITH GRANT OPTION; +REVOKE SELECT(col1) ON db.table FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--cleanup'; +REVOKE SELECT ON *.* FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--grant option 3'; +GRANT SELECT ON *.* TO test_user_01074; +REVOKE GRANT OPTION FOR SELECT(col1) ON db.table FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--cleanup'; +REVOKE SELECT ON *.* FROM test_user_01074; +SHOW GRANTS FOR test_user_01074; + +SELECT '--grant option 4'; +GRANT SELECT ON *.* TO test_user_01074; +REVOKE SELECT ON db.table FROM test_user_01074; +GRANT SELECT ON db.* TO test_user_01074 WITH GRANT OPTION; +SHOW GRANTS FOR test_user_01074; + DROP USER test_user_01074; From af3716f305b457301f0eca26d5053e403272b70e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 30 Jun 2020 19:04:15 +0300 Subject: [PATCH 18/24] Changelog for 20.3.11, 12 --- CHANGELOG.md | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a4666f08bb..606d394ff1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -323,6 +323,81 @@ ## ClickHouse release v20.3 +### ClickHouse release v20.3.12.112-lts 2020-06-25 + +#### Bug Fix + +* Fix rare crash caused by using `Nullable` column in prewhere condition. Continuation of [#11608](https://github.com/ClickHouse/ClickHouse/issues/11608). [#11869](https://github.com/ClickHouse/ClickHouse/pull/11869) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Don't allow arrayJoin inside higher order functions. It was leading to broken protocol synchronization. This closes [#3933](https://github.com/ClickHouse/ClickHouse/issues/3933). [#11846](https://github.com/ClickHouse/ClickHouse/pull/11846) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix using too many threads for queries. [#11788](https://github.com/ClickHouse/ClickHouse/pull/11788) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix unexpected behaviour of queries like `SELECT *, xyz.*` which were success while an error expected. [#11753](https://github.com/ClickHouse/ClickHouse/pull/11753) ([hexiaoting](https://github.com/hexiaoting)). +* Now replicated fetches will be cancelled during metadata alter. [#11744](https://github.com/ClickHouse/ClickHouse/pull/11744) ([alesapin](https://github.com/alesapin)). +* Fixed LOGICAL_ERROR caused by wrong type deduction of complex literals in Values input format. [#11732](https://github.com/ClickHouse/ClickHouse/pull/11732) ([tavplubix](https://github.com/tavplubix)). +* Fix `ORDER BY ... WITH FILL` over const columns. [#11697](https://github.com/ClickHouse/ClickHouse/pull/11697) ([Anton Popov](https://github.com/CurtizJ)). +* Pass proper timeouts when communicating with XDBC bridge. Recently timeouts were not respected when checking bridge liveness and receiving meta info. [#11690](https://github.com/ClickHouse/ClickHouse/pull/11690) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix error which leads to an incorrect state of `system.mutations`. It may show that whole mutation is already done but the server still has `MUTATE_PART` tasks in the replication queue and tries to execute them. This fixes [#11611](https://github.com/ClickHouse/ClickHouse/issues/11611). [#11681](https://github.com/ClickHouse/ClickHouse/pull/11681) ([alesapin](https://github.com/alesapin)). +* Add support for regular expressions with case-insensitive flags. This fixes [#11101](https://github.com/ClickHouse/ClickHouse/issues/11101) and fixes [#11506](https://github.com/ClickHouse/ClickHouse/issues/11506). [#11649](https://github.com/ClickHouse/ClickHouse/pull/11649) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Remove trivial count query optimization if row-level security is set. In previous versions the user get total count of records in a table instead filtered. This fixes [#11352](https://github.com/ClickHouse/ClickHouse/issues/11352). [#11644](https://github.com/ClickHouse/ClickHouse/pull/11644) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix bloom filters for String (data skipping indices). [#11638](https://github.com/ClickHouse/ClickHouse/pull/11638) ([Azat Khuzhin](https://github.com/azat)). +* Fix rare crash caused by using `Nullable` column in prewhere condition. (Probably it is connected with [#11572](https://github.com/ClickHouse/ClickHouse/issues/11572) somehow). [#11608](https://github.com/ClickHouse/ClickHouse/pull/11608) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix error `Block structure mismatch` for queries with sampling reading from `Buffer` table. [#11602](https://github.com/ClickHouse/ClickHouse/pull/11602) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix wrong exit code of the clickhouse-client, when exception.code() % 256 = 0. [#11601](https://github.com/ClickHouse/ClickHouse/pull/11601) ([filimonov](https://github.com/filimonov)). +* Fix trivial error in log message about "Mark cache size was lowered" at server startup. This closes [#11399](https://github.com/ClickHouse/ClickHouse/issues/11399). [#11589](https://github.com/ClickHouse/ClickHouse/pull/11589) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix error `Size of offsets doesn't match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* All queries in HTTP session have had the same query_id. It is fixed. [#11578](https://github.com/ClickHouse/ClickHouse/pull/11578) ([tavplubix](https://github.com/tavplubix)). +* Now clickhouse-server docker container will prefer IPv6 checking server aliveness. [#11550](https://github.com/ClickHouse/ClickHouse/pull/11550) ([Ivan Starkov](https://github.com/istarkov)). +* Fix shard_num/replica_num for (breaks use_compact_format_in_distributed_parts_names). [#11528](https://github.com/ClickHouse/ClickHouse/pull/11528) ([Azat Khuzhin](https://github.com/azat)). +* Fix memory leak when exception is thrown in the middle of aggregation with -State functions. This fixes [#8995](https://github.com/ClickHouse/ClickHouse/issues/8995). [#11496](https://github.com/ClickHouse/ClickHouse/pull/11496) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix wrong results of distributed queries when alias could override qualified column name. Fixes [#9672](https://github.com/ClickHouse/ClickHouse/issues/9672) [#9714](https://github.com/ClickHouse/ClickHouse/issues/9714). [#9972](https://github.com/ClickHouse/ClickHouse/pull/9972) ([Artem Zuikov](https://github.com/4ertus2)). + + +### ClickHouse release v20.3.11.97-lts 2020-06-10 + +#### New Feature + +* Now ClickHouse controls timeouts of dictionary sources on its side. Two new settings added to cache dictionary configuration: `strict_max_lifetime_seconds`, which is `max_lifetime` by default and `query_wait_timeout_milliseconds`, which is one minute by default. The first settings is also useful with `allow_read_expired_keys` settings (to forbid reading very expired keys). [#10337](https://github.com/ClickHouse/ClickHouse/pull/10337) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### Bug Fix + +* Fix the error `Data compressed with different methods` that can happen if `min_bytes_to_use_direct_io` is enabled and PREWHERE is active and using SAMPLE or high number of threads. This fixes [#11539](https://github.com/ClickHouse/ClickHouse/issues/11539). [#11540](https://github.com/ClickHouse/ClickHouse/pull/11540) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix return compressed size for codecs. [#11448](https://github.com/ClickHouse/ClickHouse/pull/11448) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix server crash when a column has compression codec with non-literal arguments. Fixes [#11365](https://github.com/ClickHouse/ClickHouse/issues/11365). [#11431](https://github.com/ClickHouse/ClickHouse/pull/11431) ([alesapin](https://github.com/alesapin)). +* Fix pointInPolygon with nan as point. Fixes https://github.com/ClickHouse/ClickHouse/issues/11375. [#11421](https://github.com/ClickHouse/ClickHouse/pull/11421) ([Alexey Ilyukhov](https://github.com/livace)). +* Fix crash in JOIN over LowCarinality(T) and Nullable(T). [#11380](https://github.com/ClickHouse/ClickHouse/issues/11380). [#11414](https://github.com/ClickHouse/ClickHouse/pull/11414) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix error code for wrong `USING` key. [#11373](https://github.com/ClickHouse/ClickHouse/issues/11373). [#11404](https://github.com/ClickHouse/ClickHouse/pull/11404) ([Artem Zuikov](https://github.com/4ertus2)). +* Fixed geohashesInBox with arguments outside of latitude/longitude range. [#11403](https://github.com/ClickHouse/ClickHouse/pull/11403) ([Vasily Nemkov](https://github.com/Enmk)). +* Better errors for `joinGet()` functions. [#11389](https://github.com/ClickHouse/ClickHouse/pull/11389) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix possible `Pipeline stuck` error for queries with external sort and limit. Fixes [#11359](https://github.com/ClickHouse/ClickHouse/issues/11359). [#11366](https://github.com/ClickHouse/ClickHouse/pull/11366) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove redundant lock during parts send in ReplicatedMergeTree. [#11354](https://github.com/ClickHouse/ClickHouse/pull/11354) ([alesapin](https://github.com/alesapin)). +* Fix support for `\G` (vertical output) in clickhouse-client in multiline mode. This closes [#9933](https://github.com/ClickHouse/ClickHouse/issues/9933). [#11350](https://github.com/ClickHouse/ClickHouse/pull/11350) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix crash in direct selects from StorageJoin (without JOIN) and wrong nullability. [#11340](https://github.com/ClickHouse/ClickHouse/pull/11340) ([Artem Zuikov](https://github.com/4ertus2)). +* Fix crash in `quantilesExactWeightedArray`. [#11337](https://github.com/ClickHouse/ClickHouse/pull/11337) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Now merges stopped before change metadata in `ALTER` queries. [#11335](https://github.com/ClickHouse/ClickHouse/pull/11335) ([alesapin](https://github.com/alesapin)). +* Make writing to `MATERIALIZED VIEW` with setting `parallel_view_processing = 1` parallel again. Fixes [#10241](https://github.com/ClickHouse/ClickHouse/issues/10241). [#11330](https://github.com/ClickHouse/ClickHouse/pull/11330) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix visitParamExtractRaw when extracted JSON has strings with unbalanced { or [. [#11318](https://github.com/ClickHouse/ClickHouse/pull/11318) ([Ewout](https://github.com/devwout)). +* Fix very rare race condition in ThreadPool. [#11314](https://github.com/ClickHouse/ClickHouse/pull/11314) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix potential uninitialized memory in conversion. Example: `SELECT toIntervalSecond(now64())`. [#11311](https://github.com/ClickHouse/ClickHouse/pull/11311) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix the issue when index analysis cannot work if a table has Array column in primary key and if a query is filtering by this column with `empty` or `notEmpty` functions. This fixes [#11286](https://github.com/ClickHouse/ClickHouse/issues/11286). [#11303](https://github.com/ClickHouse/ClickHouse/pull/11303) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix bug when query speed estimation can be incorrect and the limit of `min_execution_speed` may not work or work incorrectly if the query is throttled by `max_network_bandwidth`, `max_execution_speed` or `priority` settings. Change the default value of `timeout_before_checking_execution_speed` to non-zero, because otherwise the settings `min_execution_speed` and `max_execution_speed` have no effect. This fixes [#11297](https://github.com/ClickHouse/ClickHouse/issues/11297). This fixes [#5732](https://github.com/ClickHouse/ClickHouse/issues/5732). This fixes [#6228](https://github.com/ClickHouse/ClickHouse/issues/6228). Usability improvement: avoid concatenation of exception message with progress bar in `clickhouse-client`. [#11296](https://github.com/ClickHouse/ClickHouse/pull/11296) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix crash while reading malformed data in Protobuf format. This fixes https://github.com/ClickHouse/ClickHouse/issues/5957, fixes https://github.com/ClickHouse/ClickHouse/issues/11203. [#11258](https://github.com/ClickHouse/ClickHouse/pull/11258) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed a bug when cache-dictionary could return default value instead of normal (when there are only expired keys). This affects only string fields. [#11233](https://github.com/ClickHouse/ClickHouse/pull/11233) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix error `Block structure mismatch in QueryPipeline` while reading from `VIEW` with constants in inner query. Fixes [#11181](https://github.com/ClickHouse/ClickHouse/issues/11181). [#11205](https://github.com/ClickHouse/ClickHouse/pull/11205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible exception `Invalid status for associated output`. [#11200](https://github.com/ClickHouse/ClickHouse/pull/11200) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible error `Cannot capture column` for higher-order functions with `Array(Array(LowCardinality))` captured argument. [#11185](https://github.com/ClickHouse/ClickHouse/pull/11185) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed S3 globbing which could fail in case of more than 1000 keys and some backends. [#11179](https://github.com/ClickHouse/ClickHouse/pull/11179) ([Vladimir Chebotarev](https://github.com/excitoon)). +* If data skipping index is dependent on columns that are going to be modified during background merge (for SummingMergeTree, AggregatingMergeTree as well as for TTL GROUP BY), it was calculated incorrectly. This issue is fixed by moving index calculation after merge so the index is calculated on merged data. [#11162](https://github.com/ClickHouse/ClickHouse/pull/11162) ([Azat Khuzhin](https://github.com/azat)). +* Fix excessive reserving of threads for simple queries (optimization for reducing the number of threads, which was partly broken after changes in pipeline). [#11114](https://github.com/ClickHouse/ClickHouse/pull/11114) ([Azat Khuzhin](https://github.com/azat)). +* Fix predicates optimization for distributed queries (`enable_optimize_predicate_expression=1`) for queries with `HAVING` section (i.e. when filtering on the server initiator is required), by preserving the order of expressions (and this is enough to fix), and also force aggregator use column names over indexes. Fixes: [#10613](https://github.com/ClickHouse/ClickHouse/issues/10613), [#11413](https://github.com/ClickHouse/ClickHouse/issues/11413). [#10621](https://github.com/ClickHouse/ClickHouse/pull/10621) ([Azat Khuzhin](https://github.com/azat)). +* Introduce commit retry logic to decrease the possibility of getting duplicates from Kafka in rare cases when offset commit was failed. [#9884](https://github.com/ClickHouse/ClickHouse/pull/9884) ([filimonov](https://github.com/filimonov)). + +#### Performance Improvement + +* Get dictionary and check access rights only once per each call of any function reading external dictionaries. [#10928](https://github.com/ClickHouse/ClickHouse/pull/10928) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Build/Testing/Packaging Improvement + +* Fix several flaky integration tests. [#11355](https://github.com/ClickHouse/ClickHouse/pull/11355) ([alesapin](https://github.com/alesapin)). + ### ClickHouse release v20.3.10.75-lts 2020-05-23 #### Bug Fix From 444a869d7820746fcf44b073d47931e22a9dec87 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 30 Jun 2020 19:07:50 +0300 Subject: [PATCH 19/24] simple changelog script --- utils/simple-backport/changelog.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/simple-backport/changelog.sh b/utils/simple-backport/changelog.sh index 75a54a50b92..4b898f4c75c 100755 --- a/utils/simple-backport/changelog.sh +++ b/utils/simple-backport/changelog.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + from="$1" to="$2" log_command=(git log "$from..$to" --first-parent) @@ -82,5 +84,5 @@ done echo "### ClickHouse release $to FIXME as compared to $from " > changelog.md -./format-changelog.py changelog-prs-filtered.txt >> changelog.md +"$script_dir/format-changelog.py" changelog-prs-filtered.txt >> changelog.md cat changelog.md From e48b6b8f84426e856f2dcbb30dd77a68f39a5c56 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 30 Jun 2020 21:22:07 +0300 Subject: [PATCH 20/24] [blog] add RSS feed (#12064) * [blog] add rss feed * better title --- docs/tools/blog.py | 10 ++++++++- docs/tools/mdx_clickhouse.py | 9 ++------ docs/tools/util.py | 34 ++++++++++++++++++++++++++++++ docs/tools/website.py | 20 ++---------------- website/templates/blog/rss.xml | 23 ++++++++++++++++++++ website/templates/common_meta.html | 6 +++++- 6 files changed, 75 insertions(+), 27 deletions(-) create mode 100644 website/templates/blog/rss.xml diff --git a/docs/tools/blog.py b/docs/tools/blog.py index f5415bec608..c3261f61d4d 100644 --- a/docs/tools/blog.py +++ b/docs/tools/blog.py @@ -80,7 +80,8 @@ def build_for_lang(lang, args): includes_dir=os.path.join(os.path.dirname(__file__), '..', '_includes'), is_amp=False, is_blog=True, - post_meta=post_meta + post_meta=post_meta, + today=datetime.date.today().isoformat() ) ) @@ -89,6 +90,13 @@ def build_for_lang(lang, args): redirects.build_blog_redirects(args) + env = util.init_jinja2_env(args) + with open(os.path.join(args.website_dir, 'templates', 'blog', 'rss.xml'), 'rb') as f: + rss_template_string = f.read().decode('utf-8').strip() + rss_template = env.from_string(rss_template_string) + with open(os.path.join(args.blog_output_dir, lang, 'rss.xml'), 'w') as f: + f.write(rss_template.render({'config': raw_config})) + # TODO: AMP for blog # if not args.skip_amp: # amp.build_amp(lang, args, cfg) diff --git a/docs/tools/mdx_clickhouse.py b/docs/tools/mdx_clickhouse.py index 0c431fec106..80ecf829341 100755 --- a/docs/tools/mdx_clickhouse.py +++ b/docs/tools/mdx_clickhouse.py @@ -14,9 +14,6 @@ import macros.plugin import slugify as slugify_impl -import amp -import website - def slugify(value, separator): return slugify_impl.slugify(value, separator=separator, word_boundary=True, save_order=True) @@ -119,6 +116,7 @@ class PatchedMacrosPlugin(macros.plugin.MacrosPlugin): ]) def on_env(self, env, config, files): + import util env.add_extension('jinja2.ext.i18n') dirname = os.path.join(config.data['theme'].dirs[0], 'locale') lang = config.data['theme']['language'] @@ -126,10 +124,7 @@ class PatchedMacrosPlugin(macros.plugin.MacrosPlugin): get_translations(dirname, lang), newstyle=True ) - chunk_size = 10240 - env.filters['chunks'] = lambda line: [line[i:i+chunk_size] for i in range(0, len(line), chunk_size)] - env.filters['html_to_amp'] = amp.html_to_amp - env.filters['adjust_markdown_html'] = website.adjust_markdown_html + util.init_jinja2_filters(env) return env def render(self, markdown): diff --git a/docs/tools/util.py b/docs/tools/util.py index a5a751020f0..b840dc1168a 100644 --- a/docs/tools/util.py +++ b/docs/tools/util.py @@ -1,5 +1,6 @@ import collections import contextlib +import datetime import multiprocessing import os import shutil @@ -8,6 +9,7 @@ import socket import tempfile import threading +import jinja2 import yaml @@ -111,3 +113,35 @@ def represent_ordereddict(dumper, data): yaml.add_representer(collections.OrderedDict, represent_ordereddict) + + +def init_jinja2_filters(env): + import amp + import website + chunk_size = 10240 + env.filters['chunks'] = lambda line: [line[i:i + chunk_size] for i in range(0, len(line), chunk_size)] + env.filters['html_to_amp'] = amp.html_to_amp + env.filters['adjust_markdown_html'] = website.adjust_markdown_html + env.filters['to_rfc882'] = lambda d: datetime.datetime.strptime(d, '%Y-%m-%d').strftime('%a, %d %b %Y %H:%M:%S GMT') + + +def init_jinja2_env(args): + import mdx_clickhouse + env = jinja2.Environment( + loader=jinja2.FileSystemLoader([ + args.website_dir, + os.path.join(args.docs_dir, '_includes') + ]), + extensions=[ + 'jinja2.ext.i18n', + 'jinja2_highlight.HighlightExtension' + ] + ) + env.extend(jinja2_highlight_cssclass='syntax p-3 my-3') + translations_dir = os.path.join(args.website_dir, 'locale') + env.install_gettext_translations( + mdx_clickhouse.get_translations(translations_dir, 'en'), + newstyle=True + ) + init_jinja2_filters(env) + return env diff --git a/docs/tools/website.py b/docs/tools/website.py index d69371665ce..97d699b9916 100644 --- a/docs/tools/website.py +++ b/docs/tools/website.py @@ -11,10 +11,9 @@ import bs4 import closure import cssmin import htmlmin -import jinja2 import jsmin -import mdx_clickhouse +import util def handle_iframe(iframe, soup): @@ -121,22 +120,7 @@ def minify_html(content): def build_website(args): logging.info('Building website') - env = jinja2.Environment( - loader=jinja2.FileSystemLoader([ - args.website_dir, - os.path.join(args.docs_dir, '_includes') - ]), - extensions=[ - 'jinja2.ext.i18n', - 'jinja2_highlight.HighlightExtension' - ] - ) - env.extend(jinja2_highlight_cssclass='syntax p-3 my-3') - translations_dir = os.path.join(args.website_dir, 'locale') - env.install_gettext_translations( - mdx_clickhouse.get_translations(translations_dir, 'en'), - newstyle=True - ) + env = util.init_jinja2_env(args) shutil.copytree( args.website_dir, diff --git a/website/templates/blog/rss.xml b/website/templates/blog/rss.xml new file mode 100644 index 00000000000..1f584b522aa --- /dev/null +++ b/website/templates/blog/rss.xml @@ -0,0 +1,23 @@ + + + {{ config.site_name }} + {{ config.site_url }} + + + {{ config.extra.today|to_rfc882 }} + + {% for post in config.extra.post_meta.values() %} + {% set url = config.extra.website_url + post['url'] %} + + {{ post['title'] }} + ]]> + {{ post['date']|to_rfc882 }} + {{ url }} + {{ url }} + {# TODO: #} + + {% endfor %} + + diff --git a/website/templates/common_meta.html b/website/templates/common_meta.html index 11a36414cd7..350bcf18f05 100644 --- a/website/templates/common_meta.html +++ b/website/templates/common_meta.html @@ -26,7 +26,7 @@ {% if page and page.meta.tags %} + content="{% for tag in page.meta.tags %}{{tag}}{{ ', ' if not loop.last }}{% endfor %}" /> {% else %} @@ -45,3 +45,7 @@ {% for prefetch_item in prefetch_items %} {% endfor %} + +{% if is_blog %} + +{% endif %} From 594fc9247b6ea96925d0bac9118ff2a008c69822 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 30 Jun 2020 22:38:29 +0300 Subject: [PATCH 21/24] Update CHANGELOG.md Co-authored-by: Ivan Blinkov --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 606d394ff1e..1cb41d335bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -346,7 +346,7 @@ * Fix error `Size of offsets doesn't match size of column` for queries with `PREWHERE column in (subquery)` and `ARRAY JOIN`. [#11580](https://github.com/ClickHouse/ClickHouse/pull/11580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * All queries in HTTP session have had the same query_id. It is fixed. [#11578](https://github.com/ClickHouse/ClickHouse/pull/11578) ([tavplubix](https://github.com/tavplubix)). * Now clickhouse-server docker container will prefer IPv6 checking server aliveness. [#11550](https://github.com/ClickHouse/ClickHouse/pull/11550) ([Ivan Starkov](https://github.com/istarkov)). -* Fix shard_num/replica_num for (breaks use_compact_format_in_distributed_parts_names). [#11528](https://github.com/ClickHouse/ClickHouse/pull/11528) ([Azat Khuzhin](https://github.com/azat)). +* Fix shard_num/replica_num for `` (breaks use_compact_format_in_distributed_parts_names). [#11528](https://github.com/ClickHouse/ClickHouse/pull/11528) ([Azat Khuzhin](https://github.com/azat)). * Fix memory leak when exception is thrown in the middle of aggregation with -State functions. This fixes [#8995](https://github.com/ClickHouse/ClickHouse/issues/8995). [#11496](https://github.com/ClickHouse/ClickHouse/pull/11496) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fix wrong results of distributed queries when alias could override qualified column name. Fixes [#9672](https://github.com/ClickHouse/ClickHouse/issues/9672) [#9714](https://github.com/ClickHouse/ClickHouse/issues/9714). [#9972](https://github.com/ClickHouse/ClickHouse/pull/9972) ([Artem Zuikov](https://github.com/4ertus2)). From 03a643e9d3f5b1ffaf827284802409f5b30c96c5 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Wed, 1 Jul 2020 16:08:16 +0300 Subject: [PATCH 22/24] DOCS-609: max_server_memory_usage (#11771) * Revolg DOCSUP-1000 add max server memory usage setting (#125) * Add max_server_memory_usage setting, delete max_memory_usage_for_all_queries setting. * Syntax fixed * Apply suggestions from code review Co-authored-by: BayoNet * Doc for the max_server_memory_usage setting. Updates. Co-authored-by: Olga Revyakina Co-authored-by: BayoNet * CLICKHOUSEDOCS-609: Minor fixes. * CLICKHOUSEDOCS-609: Actualized position of the setting. Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> Co-authored-by: Olga Revyakina Co-authored-by: Sergei Shtykov --- .../settings.md | 21 +++++++++++++++++++ .../operations/settings/query-complexity.md | 9 +------- .../settings.md | 19 +++++++++++++++++ .../operations/settings/query-complexity.md | 12 ++--------- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index f90b418b4a9..f1ffc011776 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -398,6 +398,27 @@ The cache is shared for the server and memory is allocated as needed. The cache 5368709120 ``` + +## max_server_memory_usage {#max_server_memory_usage} + +Limits total RAM usage by the ClickHouse server. You can specify it only for the default profile. + +Possible values: + +- Positive integer. +- 0 — Unlimited. + +Default value: `0`. + +**Additional Info** + +On hosts with low RAM and swap, you possibly need setting `max_server_memory_usage_to_ram_ratio > 1`. + +**See also** + +- [max_memory_usage](../settings/query-complexity.md#settings_max_memory_usage) + + ## max\_concurrent\_queries {#max-concurrent-queries} The maximum number of simultaneously processed requests. diff --git a/docs/en/operations/settings/query-complexity.md b/docs/en/operations/settings/query-complexity.md index 812056785da..899f236177a 100644 --- a/docs/en/operations/settings/query-complexity.md +++ b/docs/en/operations/settings/query-complexity.md @@ -36,7 +36,7 @@ Memory usage is not monitored for the states of certain aggregate functions. Memory usage is not fully tracked for states of the aggregate functions `min`, `max`, `any`, `anyLast`, `argMin`, `argMax` from `String` and `Array` arguments. -Memory consumption is also restricted by the parameters `max_memory_usage_for_user` and `max_memory_usage_for_all_queries`. +Memory consumption is also restricted by the parameters `max_memory_usage_for_user` and [max_server_memory_usage](../server-configuration-parameters/settings.md#max_server_memory_usage). ## max\_memory\_usage\_for\_user {#max-memory-usage-for-user} @@ -46,13 +46,6 @@ Default values are defined in [Settings.h](https://github.com/ClickHouse/ClickHo See also the description of [max\_memory\_usage](#settings_max_memory_usage). -## max\_memory\_usage\_for\_all\_queries {#max-memory-usage-for-all-queries} - -The maximum amount of RAM to use for running all queries on a single server. - -Default values are defined in [Settings.h](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h#L289). By default, the amount is not restricted (`max_memory_usage_for_all_queries = 0`). - -See also the description of [max\_memory\_usage](#settings_max_memory_usage). ## max\_rows\_to\_read {#max-rows-to-read} diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 5bfedf4c520..2c933d87157 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -372,6 +372,25 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat 100 ``` +## max_server_memory_usage {#max_server_memory_usage} + +Ограничивает объём оперативной памяти, используемой сервером ClickHouse. Настройка может быть задана только для профиля `default`. + +Возможные значения: + +- Положительное целое число. +- 0 — объём используемой памяти не ограничен. + +Значение по умолчанию: `0`. + +**Дополнительная информация** + +На серверах с небольшим объёмом RAM и файла подкачки может потребоваться настройка `max_server_memory_usage_to_ram_ratio > 1`. + +**См. также** + +- [max_memory_usage](../settings/query-complexity.md#settings_max_memory_usage) + ## max\_connections {#max-connections} Максимальное количество входящих соединений. diff --git a/docs/ru/operations/settings/query-complexity.md b/docs/ru/operations/settings/query-complexity.md index 651f597c4d2..95b5a0adb63 100644 --- a/docs/ru/operations/settings/query-complexity.md +++ b/docs/ru/operations/settings/query-complexity.md @@ -1,4 +1,4 @@ -# Ограничения на сложность запроса {#ogranicheniia-na-slozhnost-zaprosa} +# Ограничения на сложность запроса {#restrictions-on-query-complexity} Ограничения на сложность запроса - часть настроек. Используются, чтобы обеспечить более безопасное исполнение запросов из пользовательского интерфейса. @@ -32,7 +32,7 @@ Потребление памяти не полностью учитывается для состояний агрегатных функций `min`, `max`, `any`, `anyLast`, `argMin`, `argMax` от аргументов `String` и `Array`. -Потребление памяти ограничивается также параметрами `max_memory_usage_for_user` и `max_memory_usage_for_all_queries`. +Потребление памяти ограничивается также параметрами `max_memory_usage_for_user` и [max_server_memory_usage](../server-configuration-parameters/settings.md#max_server_memory_usage). ## max\_memory\_usage\_for\_user {#max-memory-usage-for-user} @@ -42,14 +42,6 @@ Смотрите также описание настройки [max\_memory\_usage](#settings_max_memory_usage). -## max\_memory\_usage\_for\_all\_queries {#max-memory-usage-for-all-queries} - -Максимальный возможный объём оперативной памяти для всех запросов на одном сервере. - -Значения по умолчанию определены в файле [Settings.h](https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h#L289). По умолчанию размер не ограничен (`max_memory_usage_for_all_queries = 0`). - -Смотрите также описание настройки [max\_memory\_usage](#settings_max_memory_usage). - ## max\_rows\_to\_read {#max-rows-to-read} Следующие ограничения могут проверяться на каждый блок (а не на каждую строку). То есть, ограничения могут быть немного нарушены. From 41fed2840383e71a7eff7380daa2f274946558b7 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Wed, 1 Jul 2020 16:36:41 +0300 Subject: [PATCH 23/24] DOCS-510: runningAccumulate (#12061) * asiana21-DOCSUP-797 (#117) * docs(runningAccumulate): the function description is added * docs(runningAccumulate): the function description is modified * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * docs(runningAccumulate): some changes * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: BayoNet * docs(runningAccumulate): added ru translation Co-authored-by: asiana21 Co-authored-by: BayoNet * CLICKHOUSEDOCS-510: Minor fix. * Update docs/en/sql-reference/functions/other-functions.md Co-authored-by: Ivan Blinkov * CLICKHOUSEDOCS-510: Fixed links. Co-authored-by: AsiaKorushkina <43650329+AsiaKorushkina@users.noreply.github.com> Co-authored-by: asiana21 Co-authored-by: Sergei Shtykov Co-authored-by: Ivan Blinkov --- docs/en/development/architecture.md | 2 +- .../aggregate-functions/combinators.md | 2 +- .../functions/other-functions.md | 107 +++++++++++++++++- .../aggregate-functions/combinators.md | 2 +- .../functions/other-functions.md | 107 +++++++++++++++++- 5 files changed, 210 insertions(+), 10 deletions(-) diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md index 4c98ec12541..c2f7a86fc9f 100644 --- a/docs/en/development/architecture.md +++ b/docs/en/development/architecture.md @@ -120,7 +120,7 @@ There are ordinary functions and aggregate functions. For aggregate functions, s Ordinary functions don’t change the number of rows – they work as if they are processing each row independently. In fact, functions are not called for individual rows, but for `Block`’s of data to implement vectorized query execution. -There are some miscellaneous functions, like [blockSize](../sql-reference/functions/other-functions.md#function-blocksize), [rowNumberInBlock](../sql-reference/functions/other-functions.md#function-rownumberinblock), and [runningAccumulate](../sql-reference/functions/other-functions.md#function-runningaccumulate), that exploit block processing and violate the independence of rows. +There are some miscellaneous functions, like [blockSize](../sql-reference/functions/other-functions.md#function-blocksize), [rowNumberInBlock](../sql-reference/functions/other-functions.md#function-rownumberinblock), and [runningAccumulate](../sql-reference/functions/other-functions.md#runningaccumulatexploit block processing and violate the independence of rows. ClickHouse has strong typing, so there’s no implicit type conversion. If a function doesn't support a specific combination of types, it throws an exception. But functions can work (be overloaded) for many different combinations of types. For example, the `plus` function (to implement the `+` operator) works for any combination of numeric types: `UInt8` + `Float32`, `UInt16` + `Int8`, and so on. Also, some variadic functions can accept any number of arguments, such as the `concat` function. diff --git a/docs/en/sql-reference/aggregate-functions/combinators.md b/docs/en/sql-reference/aggregate-functions/combinators.md index 6d70637236b..554969b80a3 100644 --- a/docs/en/sql-reference/aggregate-functions/combinators.md +++ b/docs/en/sql-reference/aggregate-functions/combinators.md @@ -33,7 +33,7 @@ To work with these states, use: - [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) table engine. - [finalizeAggregation](../../sql-reference/functions/other-functions.md#function-finalizeaggregation) function. -- [runningAccumulate](../../sql-reference/functions/other-functions.md#function-runningaccumulate) function. +- [runningAccumulate](../../sql-reference/functions/other-functions.md#runningaccumulate) function. - [-Merge](#aggregate_functions_combinators-merge) combinator. - [-MergeState](#aggregate_functions_combinators-mergestate) combinator. diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index e979aff2ee7..55ddc5c029c 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -1054,11 +1054,110 @@ Result: Takes state of aggregate function. Returns result of aggregation (finalized state). -## runningAccumulate {#function-runningaccumulate} +## runningAccumulate {#runningaccumulate} -Takes the states of the aggregate function and returns a column with values, are the result of the accumulation of these states for a set of block lines, from the first to the current line. -For example, takes state of aggregate function (example runningAccumulate(uniqState(UserID))), and for each row of block, return result of aggregate function on merge of states of all previous rows and current row. -So, result of function depends on partition of data to blocks and on order of data in block. +Accumulates states of an aggregate function for each row of a data block. + +!!! warning "Warning" + The state is reset for each new data block. + +**Syntax** + +```sql +runningAccumulate(agg_state[, grouping]); +``` + +**Parameters** + +- `agg_state` — State of the aggregate function. [AggregateFunction](../../sql-reference/data-types/aggregatefunction.md#data-type-aggregatefunction). +- `grouping` — Grouping key. Optional. The state of the function is reset if the `grouping` value is changed. It can be any of the [supported data types](../../sql-reference/data-types/index.md) for which the equality operator is defined. + +**Returned value** + +- Each resulting row contains a result of the aggregate function, accumulated for all the input rows from 0 to the current position. `runningAccumulate` resets states for each new data block or when the `grouping` value changes. + +Type depends on the aggregate function used. + +**Examples** + +Consider how you can use `runningAccumulate` to find the cumulative sum of numbers without and with grouping. + +Query: + +```sql +SELECT k, runningAccumulate(sum_k) AS res FROM (SELECT number as k, sumState(k) AS sum_k FROM numbers(10) GROUP BY k ORDER BY k); +``` + +Result: + +```text +┌─k─┬─res─┐ +│ 0 │ 0 │ +│ 1 │ 1 │ +│ 2 │ 3 │ +│ 3 │ 6 │ +│ 4 │ 10 │ +│ 5 │ 15 │ +│ 6 │ 21 │ +│ 7 │ 28 │ +│ 8 │ 36 │ +│ 9 │ 45 │ +└───┴─────┘ +``` + +The subquery generates `sumState` for every number from `0` to `9`. `sumState` returns the state of the [sum](../aggregate-functions/reference/sum.md) function that contains the sum of a single number. + +The whole query does the following: + +1. For the first row, `runningAccumulate` takes `sumState(0)` and returns `0`. +2. For the second row, the function merges `sumState(0)` and `sumState(1)` resulting in `sumState(0 + 1)`, and returns `1` as a result. +3. For the third row, the function merges `sumState(0 + 1)` and `sumState(2)` resulting in `sumState(0 + 1 + 2)`, and returns `3` as a result. +4. The actions are repeated until the block ends. + +The following example shows the `groupping` parameter usage: + +Query: + +```sql +SELECT + grouping, + item, + runningAccumulate(state, grouping) AS res +FROM +( + SELECT + toInt8(number / 4) AS grouping, + number AS item, + sumState(number) AS state + FROM numbers(15) + GROUP BY item + ORDER BY item ASC +); +``` + +Result: + +```text +┌─grouping─┬─item─┬─res─┐ +│ 0 │ 0 │ 0 │ +│ 0 │ 1 │ 1 │ +│ 0 │ 2 │ 3 │ +│ 0 │ 3 │ 6 │ +│ 1 │ 4 │ 4 │ +│ 1 │ 5 │ 9 │ +│ 1 │ 6 │ 15 │ +│ 1 │ 7 │ 22 │ +│ 2 │ 8 │ 8 │ +│ 2 │ 9 │ 17 │ +│ 2 │ 10 │ 27 │ +│ 2 │ 11 │ 38 │ +│ 3 │ 12 │ 12 │ +│ 3 │ 13 │ 25 │ +│ 3 │ 14 │ 39 │ +└──────────┴──────┴─────┘ +``` + +As you can see, `runningAccumulate` merges states for each group of rows separately. ## joinGet {#joinget} diff --git a/docs/ru/sql-reference/aggregate-functions/combinators.md b/docs/ru/sql-reference/aggregate-functions/combinators.md index 95264976857..ec325d62b02 100644 --- a/docs/ru/sql-reference/aggregate-functions/combinators.md +++ b/docs/ru/sql-reference/aggregate-functions/combinators.md @@ -29,7 +29,7 @@ - Движок таблиц [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md). - Функция [finalizeAggregation](../../sql-reference/aggregate-functions/combinators.md#function-finalizeaggregation). -- Функция [runningAccumulate](../../sql-reference/aggregate-functions/combinators.md#function-runningaccumulate). +- Функция [runningAccumulate](../../sql-reference/aggregate-functions/combinators.md#runningaccumulate). - Комбинатор [-Merge](#aggregate_functions_combinators-merge). - Комбинатор [-MergeState](#aggregate_functions_combinators-mergestate). diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 0a78de66a2d..c6648963dad 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -1,4 +1,4 @@ -# Прочие функции {#prochie-funktsii} +# Прочие функции {#other-functions} ## hostName() {#hostname} @@ -1036,9 +1036,110 @@ SELECT formatReadableSize(filesystemCapacity()) AS "Capacity", toTypeName(filesy Принимает состояние агрегатной функции. Возвращает результат агрегирования. -## runningAccumulate {#function-runningaccumulate} +## runningAccumulate {#runningaccumulate} -Принимает на вход состояния агрегатной функции и возвращает столбец со значениями, которые представляют собой результат мёржа этих состояний для выборки строк из блока от первой до текущей строки. Например, принимает состояние агрегатной функции (например, `runningAccumulate(uniqState(UserID))`), и для каждой строки блока возвращает результат агрегатной функции после мёржа состояний функции для всех предыдущих строк и текущей. Таким образом, результат зависит от разбиения данных по блокам и от порядка данных в блоке. +Накапливает состояния агрегатной функции для каждой строки блока данных. + +!!! warning "Warning" + Функция обнуляет состояние для каждого нового блока. + +**Синтаксис** + +```sql +runningAccumulate(agg_state[, grouping]); +``` + +**Параметры** + +- `agg_state` — Состояние агрегатной функции. [AggregateFunction](../../sql-reference/data-types/aggregatefunction.md#data-type-aggregatefunction). +- `grouping` — Ключ группировки. Опциональный параметр. Состояние функции обнуляется, если значение `grouping` меняется. Параметр может быть любого [поддерживаемого типа данных](../../sql-reference/data-types/index.md), для которого определен оператор равенства. + +**Возвращаемое значение** + +- Каждая результирующая строка содержит результат агрегатной функции, накопленный для всех входных строк от 0 до текущей позиции. `runningAccumulate` обнуляет состояния для каждого нового блока данных или при изменении значения `grouping`. + +Тип зависит от используемой агрегатной функции. + +**Примеры** + +Рассмотрим примеры использования `runningAccumulate` для нахождения кумулятивной суммы чисел без и с группировкой. + +Запрос: + +```sql +SELECT k, runningAccumulate(sum_k) AS res FROM (SELECT number as k, sumState(k) AS sum_k FROM numbers(10) GROUP BY k ORDER BY k); +``` + +Результат: + +```text +┌─k─┬─res─┐ +│ 0 │ 0 │ +│ 1 │ 1 │ +│ 2 │ 3 │ +│ 3 │ 6 │ +│ 4 │ 10 │ +│ 5 │ 15 │ +│ 6 │ 21 │ +│ 7 │ 28 │ +│ 8 │ 36 │ +│ 9 │ 45 │ +└───┴─────┘ +``` + +Подзапрос формирует `sumState` для каждого числа от `0` до `9`. `sumState` возвращает состояние функции [sum](../../sql-reference/aggregate-functions/reference.md#agg_function-sum), содержащее сумму одного числа. + +Весь запрос делает следующее: + +1. Для первой строки `runningAccumulate` берет `sumState(0)` и возвращает `0`. +2. Для второй строки функция объединяет `sumState (0)` и `sumState (1)`, что приводит к `sumState (0 + 1)`, и возвращает в результате `1`. +3. Для третьей строки функция объединяет `sumState (0 + 1)` и `sumState (2)`, что приводит к `sumState (0 + 1 + 2)`, и в результате возвращает `3`. +4. Действия повторяются до тех пор, пока не закончится блок. + +В следующем примере показано использование параметра `grouping`: + +Запрос: + +```sql +SELECT + grouping, + item, + runningAccumulate(state, grouping) AS res +FROM +( + SELECT + toInt8(number / 4) AS grouping, + number AS item, + sumState(number) AS state + FROM numbers(15) + GROUP BY item + ORDER BY item ASC +); +``` + +Результат: + +```text +┌─grouping─┬─item─┬─res─┐ +│ 0 │ 0 │ 0 │ +│ 0 │ 1 │ 1 │ +│ 0 │ 2 │ 3 │ +│ 0 │ 3 │ 6 │ +│ 1 │ 4 │ 4 │ +│ 1 │ 5 │ 9 │ +│ 1 │ 6 │ 15 │ +│ 1 │ 7 │ 22 │ +│ 2 │ 8 │ 8 │ +│ 2 │ 9 │ 17 │ +│ 2 │ 10 │ 27 │ +│ 2 │ 11 │ 38 │ +│ 3 │ 12 │ 12 │ +│ 3 │ 13 │ 25 │ +│ 3 │ 14 │ 39 │ +└──────────┴──────┴─────┘ +``` + +Как вы можете видеть, `runningAccumulate` объединяет состояния для каждой группы строк отдельно. ## joinGet {#joinget} From f0e715ade13be05e14939e708c3ecf32a80348e9 Mon Sep 17 00:00:00 2001 From: BohuTANG Date: Wed, 1 Jul 2020 22:08:01 +0800 Subject: [PATCH 24/24] [docs] Sync zh/development/build-osx.md from EN (#12071) --- docs/zh/development/build-osx.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/zh/development/build-osx.md b/docs/zh/development/build-osx.md index 2fc68f6d2fb..24923f75207 100644 --- a/docs/zh/development/build-osx.md +++ b/docs/zh/development/build-osx.md @@ -6,13 +6,13 @@ ClickHouse 支持在 Mac OS X 10.12 版本中编译。若您在用更早的操 ## 安装 Homebrew {#an-zhuang-homebrew} ``` bash -/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" ``` ## 安装编译器，工具库 {#an-zhuang-bian-yi-qi-gong-ju-ku} ``` bash -brew install cmake ninja gcc icu4c mariadb-connector-c openssl libtool gettext +$ brew install cmake ninja libtool gettext ``` ## 拉取 ClickHouse 源码 {#la-qu-clickhouse-yuan-ma} @@ -27,11 +27,11 @@ cd ClickHouse ## 编译 ClickHouse {#bian-yi-clickhouse} ``` bash -mkdir build -cd build -cmake .. -DCMAKE_CXX_COMPILER=`which g++-8` -DCMAKE_C_COMPILER=`which gcc-8` -ninja -cd .. +$ mkdir build +$ cd build +$ cmake .. -DCMAKE_CXX_COMPILER=`which clang++` -DCMAKE_C_COMPILER=`which clang` +$ ninja +$ cd .. ``` ## 注意事项 {#zhu-yi-shi-xiang}