mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Merge branch 'master' into mongodb_refactoring
This commit is contained in:
Kirill Nikiforov
commit
a6c9150621
5
.github/PULL_REQUEST_TEMPLATE.md
vendored
5
.github/PULL_REQUEST_TEMPLATE.md
vendored
@ -48,9 +48,7 @@ At a minimum, the following information should be added (but add more as needed)
|
||||
- [ ] <!---ci_include_stateful--> Allow: Stateful tests
|
||||
- [ ] <!---ci_include_integration--> Allow: Integration Tests
|
||||
- [ ] <!---ci_include_performance--> Allow: Performance tests
|
||||
- [ ] <!---ci_set_normal_builds--> Allow: Normal Builds
|
||||
- [ ] <!---ci_set_special_builds--> Allow: Special Builds
|
||||
- [ ] <!---ci_set_non_required--> Allow: All NOT Required Checks
|
||||
- [ ] <!---ci_set_builds--> Allow: All Builds
|
||||
- [ ] <!---batch_0_1--> Allow: batch 1, 2 for multi-batch jobs
|
||||
- [ ] <!---batch_2_3--> Allow: batch 3, 4, 5, 6 for multi-batch jobs
|
||||
---
|
||||
@ -61,6 +59,7 @@ At a minimum, the following information should be added (but add more as needed)
|
||||
- [ ] <!---ci_exclude_aarch64|release|debug--> Exclude: All with aarch64, release, debug
|
||||
---
|
||||
- [ ] <!---do_not_test--> Do not test
|
||||
- [ ] <!---woolen_wolfdog--> Woolen Wolfdog
|
||||
- [ ] <!---upload_all--> Upload binaries for special builds
|
||||
- [ ] <!---no_merge_commit--> Disable merge-commit
|
||||
- [ ] <!---no_ci_cache--> Disable CI cache
|
||||
|
||||
6
.github/workflows/backport_branches.yml
vendored
6
.github/workflows/backport_branches.yml
vendored
@ -70,7 +70,7 @@ jobs:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Compatibility check (amd64)
|
||||
test_name: Compatibility check (release)
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
CompatibilityCheckAarch64:
|
||||
@ -194,7 +194,7 @@ jobs:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Install packages (amd64)
|
||||
test_name: Install packages (release)
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
@ -204,7 +204,7 @@ jobs:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Install packages (arm64)
|
||||
test_name: Install packages (aarch64)
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
|
||||
20
.github/workflows/master.yml
vendored
20
.github/workflows/master.yml
vendored
@ -104,10 +104,9 @@ jobs:
|
||||
with:
|
||||
stage: Tests_2
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# stage for jobs that do not prohibit merge
|
||||
Tests_3:
|
||||
# Test_3 should not wait for Test_1/Test_2 and should not be blocked by them on master branch since all jobs need to run there.
|
||||
needs: [RunConfig, Builds_1, Builds_2]
|
||||
needs: [RunConfig, Builds_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
|
||||
uses: ./.github/workflows/reusable_test_stage.yml
|
||||
with:
|
||||
@ -115,25 +114,16 @@ jobs:
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
|
||||
################################# Reports #################################
|
||||
# Reports should be run even if Builds_1/2 failed - put them separately in wf (not in Tests_1/2)
|
||||
Builds_1_Report:
|
||||
# Reports should run even if Builds_1/2 fail - run them separately, not in Tests_1/2/3
|
||||
Builds_Report:
|
||||
# run report check for failed builds to indicate the CI error
|
||||
if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
|
||||
needs: [RunConfig, Builds_1]
|
||||
needs: [RunConfig, Builds_1, Builds_2]
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse build check
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
Builds_2_Report:
|
||||
# run report check for failed builds to indicate the CI error
|
||||
if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse special build check') }}
|
||||
needs: [RunConfig, Builds_2]
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse special build check
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
|
||||
MarkReleaseReady:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
@ -165,7 +155,7 @@ jobs:
|
||||
|
||||
FinishCheck:
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [RunConfig, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3]
|
||||
needs: [RunConfig, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
|
||||
31
.github/workflows/pull_request.yml
vendored
31
.github/workflows/pull_request.yml
vendored
@ -126,16 +126,16 @@ jobs:
|
||||
with:
|
||||
stage: Builds_2
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# stage for running non-required checks without being blocked by required checks (Test_1) if corresponding settings is selected
|
||||
Tests_2:
|
||||
needs: [RunConfig, Builds_2]
|
||||
needs: [RunConfig, Builds_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
|
||||
uses: ./.github/workflows/reusable_test_stage.yml
|
||||
with:
|
||||
stage: Tests_2
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# stage for jobs that do not prohibit merge
|
||||
Tests_3:
|
||||
needs: [RunConfig, Builds_1, Tests_1, Builds_2, Tests_2]
|
||||
needs: [RunConfig, Builds_1, Tests_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
|
||||
uses: ./.github/workflows/reusable_test_stage.yml
|
||||
with:
|
||||
@ -143,29 +143,21 @@ jobs:
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
|
||||
################################# Reports #################################
|
||||
# Reports should by run even if Builds_1/2 fail, so put them separately in wf (not in Tests_1/2)
|
||||
Builds_1_Report:
|
||||
# Reports should run even if Builds_1/2 fail - run them separately (not in Tests_1/2/3)
|
||||
Builds_Report:
|
||||
# run report check for failed builds to indicate the CI error
|
||||
if: ${{ !cancelled() && needs.StyleCheck.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
|
||||
needs: [RunConfig, StyleCheck, Builds_1]
|
||||
if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
|
||||
needs: [RunConfig, StyleCheck, Builds_1, Builds_2]
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse build check
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
Builds_2_Report:
|
||||
# run report check for failed builds to indicate the CI error
|
||||
if: ${{ !cancelled() && needs.StyleCheck.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse special build check') }}
|
||||
needs: [RunConfig, StyleCheck, Builds_2]
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse special build check
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
|
||||
CheckReadyForMerge:
|
||||
if: ${{ !cancelled() && needs.StyleCheck.result == 'success' }}
|
||||
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2]
|
||||
# Test_2 or Test_3 must not have jobs required for Mergeable check
|
||||
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1]
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
@ -181,7 +173,7 @@ jobs:
|
||||
#
|
||||
FinishCheck:
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3]
|
||||
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
@ -204,8 +196,7 @@ jobs:
|
||||
concurrency:
|
||||
group: jepsen
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse Keeper Jepsen') }}
|
||||
# jepsen needs binary_release build which is in Builds_2
|
||||
needs: [RunConfig, Builds_2]
|
||||
needs: [RunConfig, Builds_1]
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse Keeper Jepsen
|
||||
|
||||
6
.github/workflows/release_branches.yml
vendored
6
.github/workflows/release_branches.yml
vendored
@ -65,7 +65,7 @@ jobs:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Compatibility check (amd64)
|
||||
test_name: Compatibility check (release)
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
CompatibilityCheckAarch64:
|
||||
@ -244,7 +244,7 @@ jobs:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Install packages (amd64)
|
||||
test_name: Install packages (release)
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
@ -254,7 +254,7 @@ jobs:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Install packages (arm64)
|
||||
test_name: Install packages (aarch64)
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
|
||||
10
README.md
10
README.md
@ -34,20 +34,18 @@ curl https://clickhouse.com/ | sh
|
||||
|
||||
Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know.
|
||||
|
||||
* [v24.5 Community Call](https://clickhouse.com/company/events/v24-5-community-release-call) - May 30
|
||||
* [v24.6 Community Call](https://clickhouse.com/company/events/v24-6-community-release-call) - Jul 2
|
||||
|
||||
## Upcoming Events
|
||||
|
||||
Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
|
||||
|
||||
* [ClickHouse Happy Hour @ Tom's Watch Bar - Los Angeles](https://www.meetup.com/clickhouse-los-angeles-user-group/events/300740584/) - May 22
|
||||
* [ClickHouse & Confluent Meetup in Dubai](https://www.meetup.com/clickhouse-dubai-meetup-group/events/299629189/) - May 28
|
||||
* [ClickHouse Meetup in Stockholm](https://www.meetup.com/clickhouse-stockholm-user-group/events/299752651/) - Jun 3
|
||||
* [ClickHouse Meetup @ Cloudflare - San Francisco](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/300523061/) - Jun 4
|
||||
* [ClickHouse (クリックハウス) Meetup Tokyo](https://www.meetup.com/clickhouse-tokyo-user-group/events/300798053/) - Jun 5
|
||||
* [AWS Summit in DC](https://clickhouse.com/company/events/2024-06-aws-summit-dc) - Jun 26
|
||||
* [ClickHouse Meetup in Amsterdam](https://www.meetup.com/clickhouse-netherlands-user-group/events/300781068/) - Jun 27
|
||||
* [ClickHouse Meetup in Paris](https://www.meetup.com/clickhouse-france-user-group/events/300783448/) - Jul 9
|
||||
* [ClickHouse Cloud - Live Update Call](https://clickhouse.com/company/events/202407-cloud-update-live) - Jul 9
|
||||
* [ClickHouse Meetup @ Ramp - New York City](https://www.meetup.com/clickhouse-new-york-user-group/events/300595845/) - Jul 9
|
||||
* [AWS Summit in New York](https://clickhouse.com/company/events/2024-07-awssummit-nyc) - Jul 10
|
||||
* [ClickHouse Meetup @ Klaviyo - Boston](https://www.meetup.com/clickhouse-boston-user-group/events/300907870) - Jul 11
|
||||
|
||||
## Recent Recordings
|
||||
|
||||
@ -23,9 +23,6 @@
|
||||
#include <openssl/conf.h>
|
||||
#endif
|
||||
|
||||
#if __has_feature(address_sanitizer)
|
||||
#include <sanitizer/lsan_interface.h>
|
||||
#endif
|
||||
|
||||
using Poco::RandomInputStream;
|
||||
using Poco::Thread;
|
||||
@ -70,18 +67,12 @@ void OpenSSLInitializer::initialize()
|
||||
SSL_library_init();
|
||||
SSL_load_error_strings();
|
||||
OpenSSL_add_all_algorithms();
|
||||
|
||||
|
||||
char seed[SEEDSIZE];
|
||||
RandomInputStream rnd;
|
||||
rnd.read(seed, sizeof(seed));
|
||||
{
|
||||
# if __has_feature(address_sanitizer)
|
||||
/// Leak sanitizer (part of address sanitizer) thinks that a few bytes of memory in OpenSSL are allocated during but never released.
|
||||
__lsan::ScopedDisabler lsan_disabler;
|
||||
#endif
|
||||
RAND_seed(seed, SEEDSIZE);
|
||||
}
|
||||
|
||||
RAND_seed(seed, SEEDSIZE);
|
||||
|
||||
int nMutexes = CRYPTO_num_locks();
|
||||
_mutexes = new Poco::FastMutex[nMutexes];
|
||||
CRYPTO_set_locking_callback(&OpenSSLInitializer::lock);
|
||||
@ -89,8 +80,8 @@ void OpenSSLInitializer::initialize()
|
||||
// https://sourceforge.net/p/poco/bugs/110/
|
||||
//
|
||||
// From http://www.openssl.org/docs/crypto/threads.html :
|
||||
// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(),
|
||||
// then a default implementation is used - on Windows and BeOS this uses the system's
|
||||
// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(),
|
||||
// then a default implementation is used - on Windows and BeOS this uses the system's
|
||||
// default thread identifying APIs"
|
||||
CRYPTO_set_id_callback(&OpenSSLInitializer::id);
|
||||
CRYPTO_set_dynlock_create_callback(&OpenSSLInitializer::dynlockCreate);
|
||||
@ -109,7 +100,7 @@ void OpenSSLInitializer::uninitialize()
|
||||
CRYPTO_set_locking_callback(0);
|
||||
CRYPTO_set_id_callback(0);
|
||||
delete [] _mutexes;
|
||||
|
||||
|
||||
CONF_modules_free();
|
||||
}
|
||||
}
|
||||
|
||||
@ -213,6 +213,7 @@ target_compile_definitions (_poco_foundation
|
||||
)
|
||||
|
||||
target_include_directories (_poco_foundation SYSTEM PUBLIC "include")
|
||||
target_link_libraries (_poco_foundation PRIVATE clickhouse_common_io)
|
||||
|
||||
target_link_libraries (_poco_foundation
|
||||
PRIVATE
|
||||
|
||||
@ -48,7 +48,13 @@ class Foundation_API ThreadPool
|
||||
/// from the pool.
|
||||
{
|
||||
public:
|
||||
ThreadPool(int minCapacity = 2, int maxCapacity = 16, int idleTime = 60, int stackSize = POCO_THREAD_STACK_SIZE);
|
||||
explicit ThreadPool(
|
||||
int minCapacity = 2,
|
||||
int maxCapacity = 16,
|
||||
int idleTime = 60,
|
||||
int stackSize = POCO_THREAD_STACK_SIZE,
|
||||
size_t global_profiler_real_time_period_ns_ = 0,
|
||||
size_t global_profiler_cpu_time_period_ns_ = 0);
|
||||
/// Creates a thread pool with minCapacity threads.
|
||||
/// If required, up to maxCapacity threads are created
|
||||
/// a NoThreadAvailableException exception is thrown.
|
||||
@ -56,8 +62,14 @@ public:
|
||||
/// and more than minCapacity threads are running, the thread
|
||||
/// is killed. Threads are created with given stack size.
|
||||
|
||||
ThreadPool(
|
||||
const std::string & name, int minCapacity = 2, int maxCapacity = 16, int idleTime = 60, int stackSize = POCO_THREAD_STACK_SIZE);
|
||||
explicit ThreadPool(
|
||||
const std::string & name,
|
||||
int minCapacity = 2,
|
||||
int maxCapacity = 16,
|
||||
int idleTime = 60,
|
||||
int stackSize = POCO_THREAD_STACK_SIZE,
|
||||
size_t global_profiler_real_time_period_ns_ = 0,
|
||||
size_t global_profiler_cpu_time_period_ns_ = 0);
|
||||
/// Creates a thread pool with the given name and minCapacity threads.
|
||||
/// If required, up to maxCapacity threads are created
|
||||
/// a NoThreadAvailableException exception is thrown.
|
||||
@ -171,6 +183,8 @@ private:
|
||||
int _serial;
|
||||
int _age;
|
||||
int _stackSize;
|
||||
size_t _globalProfilerRealTimePeriodNs;
|
||||
size_t _globalProfilerCPUTimePeriodNs;
|
||||
ThreadVec _threads;
|
||||
mutable FastMutex _mutex;
|
||||
};
|
||||
|
||||
@ -20,6 +20,7 @@
|
||||
#include "Poco/ErrorHandler.h"
|
||||
#include <sstream>
|
||||
#include <ctime>
|
||||
#include <Common/ThreadPool.h>
|
||||
|
||||
|
||||
namespace Poco {
|
||||
@ -28,7 +29,11 @@ namespace Poco {
|
||||
class PooledThread: public Runnable
|
||||
{
|
||||
public:
|
||||
PooledThread(const std::string& name, int stackSize = POCO_THREAD_STACK_SIZE);
|
||||
explicit PooledThread(
|
||||
const std::string& name,
|
||||
int stackSize = POCO_THREAD_STACK_SIZE,
|
||||
size_t globalProfilerRealTimePeriodNs_ = 0,
|
||||
size_t globalProfilerCPUTimePeriodNs_ = 0);
|
||||
~PooledThread();
|
||||
|
||||
void start();
|
||||
@ -51,16 +56,24 @@ private:
|
||||
Event _targetCompleted;
|
||||
Event _started;
|
||||
FastMutex _mutex;
|
||||
size_t _globalProfilerRealTimePeriodNs;
|
||||
size_t _globalProfilerCPUTimePeriodNs;
|
||||
};
|
||||
|
||||
|
||||
PooledThread::PooledThread(const std::string& name, int stackSize):
|
||||
_idle(true),
|
||||
_idleTime(0),
|
||||
_pTarget(0),
|
||||
_name(name),
|
||||
PooledThread::PooledThread(
|
||||
const std::string& name,
|
||||
int stackSize,
|
||||
size_t globalProfilerRealTimePeriodNs_,
|
||||
size_t globalProfilerCPUTimePeriodNs_) :
|
||||
_idle(true),
|
||||
_idleTime(0),
|
||||
_pTarget(0),
|
||||
_name(name),
|
||||
_thread(name),
|
||||
_targetCompleted(false)
|
||||
_targetCompleted(false),
|
||||
_globalProfilerRealTimePeriodNs(globalProfilerRealTimePeriodNs_),
|
||||
_globalProfilerCPUTimePeriodNs(globalProfilerCPUTimePeriodNs_)
|
||||
{
|
||||
poco_assert_dbg (stackSize >= 0);
|
||||
_thread.setStackSize(stackSize);
|
||||
@ -83,7 +96,7 @@ void PooledThread::start()
|
||||
void PooledThread::start(Thread::Priority priority, Runnable& target)
|
||||
{
|
||||
FastMutex::ScopedLock lock(_mutex);
|
||||
|
||||
|
||||
poco_assert (_pTarget == 0);
|
||||
|
||||
_pTarget = ⌖
|
||||
@ -109,7 +122,7 @@ void PooledThread::start(Thread::Priority priority, Runnable& target, const std:
|
||||
}
|
||||
_thread.setName(fullName);
|
||||
_thread.setPriority(priority);
|
||||
|
||||
|
||||
poco_assert (_pTarget == 0);
|
||||
|
||||
_pTarget = ⌖
|
||||
@ -145,7 +158,7 @@ void PooledThread::join()
|
||||
void PooledThread::activate()
|
||||
{
|
||||
FastMutex::ScopedLock lock(_mutex);
|
||||
|
||||
|
||||
poco_assert (_idle);
|
||||
_idle = false;
|
||||
_targetCompleted.reset();
|
||||
@ -155,7 +168,7 @@ void PooledThread::activate()
|
||||
void PooledThread::release()
|
||||
{
|
||||
const long JOIN_TIMEOUT = 10000;
|
||||
|
||||
|
||||
_mutex.lock();
|
||||
_pTarget = 0;
|
||||
_mutex.unlock();
|
||||
@ -174,6 +187,10 @@ void PooledThread::release()
|
||||
|
||||
void PooledThread::run()
|
||||
{
|
||||
DB::ThreadStatus thread_status;
|
||||
if (unlikely(_globalProfilerRealTimePeriodNs != 0 || _globalProfilerCPUTimePeriodNs != 0))
|
||||
thread_status.initGlobalProfiler(_globalProfilerRealTimePeriodNs, _globalProfilerCPUTimePeriodNs);
|
||||
|
||||
_started.set();
|
||||
for (;;)
|
||||
{
|
||||
@ -220,13 +237,17 @@ void PooledThread::run()
|
||||
ThreadPool::ThreadPool(int minCapacity,
|
||||
int maxCapacity,
|
||||
int idleTime,
|
||||
int stackSize):
|
||||
_minCapacity(minCapacity),
|
||||
_maxCapacity(maxCapacity),
|
||||
int stackSize,
|
||||
size_t globalProfilerRealTimePeriodNs_,
|
||||
size_t globalProfilerCPUTimePeriodNs_) :
|
||||
_minCapacity(minCapacity),
|
||||
_maxCapacity(maxCapacity),
|
||||
_idleTime(idleTime),
|
||||
_serial(0),
|
||||
_age(0),
|
||||
_stackSize(stackSize)
|
||||
_stackSize(stackSize),
|
||||
_globalProfilerRealTimePeriodNs(globalProfilerRealTimePeriodNs_),
|
||||
_globalProfilerCPUTimePeriodNs(globalProfilerCPUTimePeriodNs_)
|
||||
{
|
||||
poco_assert (minCapacity >= 1 && maxCapacity >= minCapacity && idleTime > 0);
|
||||
|
||||
@ -243,14 +264,18 @@ ThreadPool::ThreadPool(const std::string& name,
|
||||
int minCapacity,
|
||||
int maxCapacity,
|
||||
int idleTime,
|
||||
int stackSize):
|
||||
int stackSize,
|
||||
size_t globalProfilerRealTimePeriodNs_,
|
||||
size_t globalProfilerCPUTimePeriodNs_) :
|
||||
_name(name),
|
||||
_minCapacity(minCapacity),
|
||||
_maxCapacity(maxCapacity),
|
||||
_minCapacity(minCapacity),
|
||||
_maxCapacity(maxCapacity),
|
||||
_idleTime(idleTime),
|
||||
_serial(0),
|
||||
_age(0),
|
||||
_stackSize(stackSize)
|
||||
_stackSize(stackSize),
|
||||
_globalProfilerRealTimePeriodNs(globalProfilerRealTimePeriodNs_),
|
||||
_globalProfilerCPUTimePeriodNs(globalProfilerCPUTimePeriodNs_)
|
||||
{
|
||||
poco_assert (minCapacity >= 1 && maxCapacity >= minCapacity && idleTime > 0);
|
||||
|
||||
@ -393,15 +418,15 @@ void ThreadPool::housekeep()
|
||||
ThreadVec activeThreads;
|
||||
idleThreads.reserve(_threads.size());
|
||||
activeThreads.reserve(_threads.size());
|
||||
|
||||
|
||||
for (ThreadVec::iterator it = _threads.begin(); it != _threads.end(); ++it)
|
||||
{
|
||||
if ((*it)->idle())
|
||||
{
|
||||
if ((*it)->idleTime() < _idleTime)
|
||||
idleThreads.push_back(*it);
|
||||
else
|
||||
expiredThreads.push_back(*it);
|
||||
else
|
||||
expiredThreads.push_back(*it);
|
||||
}
|
||||
else activeThreads.push_back(*it);
|
||||
}
|
||||
@ -463,7 +488,7 @@ PooledThread* ThreadPool::createThread()
|
||||
{
|
||||
std::ostringstream name;
|
||||
name << _name << "[#" << ++_serial << "]";
|
||||
return new PooledThread(name.str(), _stackSize);
|
||||
return new PooledThread(name.str(), _stackSize, _globalProfilerRealTimePeriodNs, _globalProfilerCPUTimePeriodNs);
|
||||
}
|
||||
|
||||
|
||||
@ -481,7 +506,7 @@ public:
|
||||
ThreadPool* pool()
|
||||
{
|
||||
FastMutex::ScopedLock lock(_mutex);
|
||||
|
||||
|
||||
if (!_pPool)
|
||||
{
|
||||
_pPool = new ThreadPool("default");
|
||||
@ -490,7 +515,7 @@ public:
|
||||
}
|
||||
return _pPool;
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
ThreadPool* _pPool;
|
||||
FastMutex _mutex;
|
||||
|
||||
@ -330,27 +330,26 @@ void SSLManager::initDefaultContext(bool server)
|
||||
else
|
||||
_ptrDefaultClientContext->disableProtocols(disabledProtocols);
|
||||
|
||||
/// Temporarily disabled during the transition from boringssl to OpenSSL due to tsan issues.
|
||||
/// bool cacheSessions = config.getBool(prefix + CFG_CACHE_SESSIONS, false);
|
||||
/// if (server)
|
||||
/// {
|
||||
/// std::string sessionIdContext = config.getString(prefix + CFG_SESSION_ID_CONTEXT, config.getString("application.name", ""));
|
||||
/// _ptrDefaultServerContext->enableSessionCache(cacheSessions, sessionIdContext);
|
||||
/// if (config.hasProperty(prefix + CFG_SESSION_CACHE_SIZE))
|
||||
/// {
|
||||
/// int cacheSize = config.getInt(prefix + CFG_SESSION_CACHE_SIZE);
|
||||
/// _ptrDefaultServerContext->setSessionCacheSize(cacheSize);
|
||||
/// }
|
||||
/// if (config.hasProperty(prefix + CFG_SESSION_TIMEOUT))
|
||||
/// {
|
||||
/// int timeout = config.getInt(prefix + CFG_SESSION_TIMEOUT);
|
||||
/// _ptrDefaultServerContext->setSessionTimeout(timeout);
|
||||
/// }
|
||||
/// }
|
||||
/// else
|
||||
/// {
|
||||
/// _ptrDefaultClientContext->enableSessionCache(cacheSessions);
|
||||
/// }
|
||||
bool cacheSessions = config.getBool(prefix + CFG_CACHE_SESSIONS, false);
|
||||
if (server)
|
||||
{
|
||||
std::string sessionIdContext = config.getString(prefix + CFG_SESSION_ID_CONTEXT, config.getString("application.name", ""));
|
||||
_ptrDefaultServerContext->enableSessionCache(cacheSessions, sessionIdContext);
|
||||
if (config.hasProperty(prefix + CFG_SESSION_CACHE_SIZE))
|
||||
{
|
||||
int cacheSize = config.getInt(prefix + CFG_SESSION_CACHE_SIZE);
|
||||
_ptrDefaultServerContext->setSessionCacheSize(cacheSize);
|
||||
}
|
||||
if (config.hasProperty(prefix + CFG_SESSION_TIMEOUT))
|
||||
{
|
||||
int timeout = config.getInt(prefix + CFG_SESSION_TIMEOUT);
|
||||
_ptrDefaultServerContext->setSessionTimeout(timeout);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_ptrDefaultClientContext->enableSessionCache(cacheSessions);
|
||||
}
|
||||
bool extendedVerification = config.getBool(prefix + CFG_EXTENDED_VERIFICATION, false);
|
||||
if (server)
|
||||
_ptrDefaultServerContext->enableExtendedCertificateVerification(extendedVerification);
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
# This variables autochanged by tests/ci/version_helper.py:
|
||||
|
||||
# NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
|
||||
# NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION,
|
||||
# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
|
||||
SET(VERSION_REVISION 54487)
|
||||
SET(VERSION_REVISION 54488)
|
||||
SET(VERSION_MAJOR 24)
|
||||
SET(VERSION_MINOR 6)
|
||||
SET(VERSION_MINOR 7)
|
||||
SET(VERSION_PATCH 1)
|
||||
SET(VERSION_GITHASH 70a1d3a63d47f0be077d67b8deb907230fc7cfb0)
|
||||
SET(VERSION_DESCRIBE v24.6.1.1-testing)
|
||||
SET(VERSION_STRING 24.6.1.1)
|
||||
SET(VERSION_GITHASH aa023477a9265e403982fca5ee29a714db5133d9)
|
||||
SET(VERSION_DESCRIBE v24.7.1.1-testing)
|
||||
SET(VERSION_STRING 24.7.1.1)
|
||||
# end of autochange
|
||||
|
||||
2
contrib/openssl
vendored
2
contrib/openssl
vendored
@ -1 +1 @@
|
||||
Subproject commit 67c0b63e578e4c751ac9edf490f5a96124fff8dc
|
||||
Subproject commit 5d81fa7068fc8c07f4d0997d5b703f3c541a637c
|
||||
2
contrib/re2
vendored
2
contrib/re2
vendored
@ -1 +1 @@
|
||||
Subproject commit a807e8a3aac2cc33c77b7071efea54fcabe38e0c
|
||||
Subproject commit 85dd7ad833a73095ecf3e3baea608ba051bbe2c7
|
||||
@ -28,16 +28,20 @@ set(RE2_SOURCES
|
||||
add_library(_re2 ${RE2_SOURCES})
|
||||
target_include_directories(_re2 PUBLIC "${SRC_DIR}")
|
||||
target_link_libraries(_re2 PRIVATE
|
||||
absl::absl_check
|
||||
absl::absl_log
|
||||
absl::base
|
||||
absl::core_headers
|
||||
absl::fixed_array
|
||||
absl::flags
|
||||
absl::flat_hash_map
|
||||
absl::flat_hash_set
|
||||
absl::hash
|
||||
absl::inlined_vector
|
||||
absl::strings
|
||||
absl::str_format
|
||||
absl::synchronization
|
||||
absl::optional
|
||||
absl::span)
|
||||
absl::span
|
||||
absl::str_format
|
||||
absl::strings
|
||||
absl::synchronization)
|
||||
|
||||
add_library(ch_contrib::re2 ALIAS _re2)
|
||||
|
||||
@ -254,7 +254,7 @@ function run_tests()
|
||||
|
||||
set +e
|
||||
clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
|
||||
--test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
|
||||
--no-drop-if-fail --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
|
||||
| ts '%Y-%m-%d %H:%M:%S' \
|
||||
| tee -a test_output/test_result.txt
|
||||
set -e
|
||||
@ -285,7 +285,7 @@ stop_logs_replication
|
||||
|
||||
# Try to get logs while server is running
|
||||
failed_to_save_logs=0
|
||||
for table in query_log zookeeper_log trace_log transactions_info_log metric_log blob_storage_log
|
||||
for table in query_log zookeeper_log trace_log transactions_info_log metric_log blob_storage_log error_log
|
||||
do
|
||||
err=$(clickhouse-client -q "select * from system.$table into outfile '/test_output/$table.tsv.gz' format TSVWithNamesAndTypes")
|
||||
echo "$err"
|
||||
@ -339,7 +339,7 @@ if [ $failed_to_save_logs -ne 0 ]; then
|
||||
# directly
|
||||
# - even though ci auto-compress some files (but not *.tsv) it does this only
|
||||
# for files >64MB, we want this files to be compressed explicitly
|
||||
for table in query_log zookeeper_log trace_log transactions_info_log metric_log blob_storage_log
|
||||
for table in query_log zookeeper_log trace_log transactions_info_log metric_log blob_storage_log error_log
|
||||
do
|
||||
clickhouse-local "$data_path_config" --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst ||:
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
@ -379,6 +379,10 @@ fi
|
||||
|
||||
tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
|
||||
|
||||
rm -rf /var/lib/clickhouse/data/system/*/
|
||||
tar -chf /test_output/store.tar /var/lib/clickhouse/store ||:
|
||||
tar -chf /test_output/metadata.tar /var/lib/clickhouse/metadata/*.sql ||:
|
||||
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server1.log ||:
|
||||
rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server2.log ||:
|
||||
|
||||
@ -89,10 +89,6 @@ function configure()
|
||||
# since we run clickhouse from root
|
||||
sudo chown root: /var/lib/clickhouse
|
||||
|
||||
# Set more frequent update period of asynchronous metrics to more frequently update information about real memory usage (less chance of OOM).
|
||||
echo "<clickhouse><asynchronous_metrics_update_period_s>1</asynchronous_metrics_update_period_s></clickhouse>" \
|
||||
> /etc/clickhouse-server/config.d/asynchronous_metrics_update_period_s.xml
|
||||
|
||||
local total_mem
|
||||
total_mem=$(awk '/MemTotal/ { print $(NF-1) }' /proc/meminfo) # KiB
|
||||
total_mem=$(( total_mem*1024 )) # bytes
|
||||
|
||||
@ -37,7 +37,7 @@ Using named collections:
|
||||
<named_collections>
|
||||
<iceberg_conf>
|
||||
<url>http://test.s3.amazonaws.com/clickhouse-bucket/</url>
|
||||
<access_key_id>test<access_key_id>
|
||||
<access_key_id>test</access_key_id>
|
||||
<secret_access_key>test</secret_access_key>
|
||||
</iceberg_conf>
|
||||
</named_collections>
|
||||
|
||||
@ -13,7 +13,7 @@ This engine provides integration with [Amazon S3](https://aws.amazon.com/s3/) ec
|
||||
CREATE TABLE s3_queue_engine_table (name String, value UInt32)
|
||||
ENGINE = S3Queue(path, [NOSIGN, | aws_access_key_id, aws_secret_access_key,] format, [compression])
|
||||
[SETTINGS]
|
||||
[mode = 'unordered',]
|
||||
[mode = '',]
|
||||
[after_processing = 'keep',]
|
||||
[keeper_path = '',]
|
||||
[s3queue_loading_retries = 0,]
|
||||
|
||||
@ -1490,6 +1490,8 @@ Differs from [PrettySpaceNoEscapes](#prettyspacenoescapes) in that up to 10,000
|
||||
- [output_format_pretty_color](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_color) - use ANSI escape sequences to paint colors in Pretty formats. Default value - `true`.
|
||||
- [output_format_pretty_grid_charset](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_grid_charset) - Charset for printing grid borders. Available charsets: ASCII, UTF-8. Default value - `UTF-8`.
|
||||
- [output_format_pretty_row_numbers](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_row_numbers) - Add row numbers before each row for pretty output format. Default value - `true`.
|
||||
- [output_format_pretty_display_footer_column_names](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_display_footer_column_names) - Display column names in the footer if table contains many rows. Default value - `true`.
|
||||
- [output_format_pretty_display_footer_column_names_min_rows](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_display_footer_column_names_min_rows) - Sets the minimum number of rows for which a footer will be displayed if [output_format_pretty_display_footer_column_names](/docs/en/operations/settings/settings-formats.md/#output_format_pretty_display_footer_column_names) is enabled. Default value - 50.
|
||||
|
||||
## RowBinary {#rowbinary}
|
||||
|
||||
|
||||
@ -508,7 +508,7 @@ Now `rule` can configure `method`, `headers`, `url`, `handler`:
|
||||
|
||||
- `headers` are responsible for matching the header part of the HTTP request. It is compatible with RE2’s regular expressions. It is an optional configuration. If it is not defined in the configuration file, it does not match the header portion of the HTTP request.
|
||||
|
||||
- `handler` contains the main processing part. Now `handler` can configure `type`, `status`, `content_type`, `response_content`, `query`, `query_param_name`.
|
||||
- `handler` contains the main processing part. Now `handler` can configure `type`, `status`, `content_type`, `http_response_headers`, `response_content`, `query`, `query_param_name`.
|
||||
`type` currently supports three types: [predefined_query_handler](#predefined_query_handler), [dynamic_query_handler](#dynamic_query_handler), [static](#static).
|
||||
|
||||
- `query` — use with `predefined_query_handler` type, executes query when the handler is called.
|
||||
@ -519,6 +519,8 @@ Now `rule` can configure `method`, `headers`, `url`, `handler`:
|
||||
|
||||
- `content_type` — use with any type, response [content-type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type).
|
||||
|
||||
- `http_response_headers` — use with any type, response headers map. Could be used to set content type as well.
|
||||
|
||||
- `response_content` — use with `static` type, response content sent to client, when using the prefix ‘file://’ or ‘config://’, find the content from the file or configuration sends to client.
|
||||
|
||||
Next are the configuration methods for different `type`.
|
||||
@ -616,6 +618,33 @@ Return a message.
|
||||
<type>static</type>
|
||||
<status>402</status>
|
||||
<content_type>text/html; charset=UTF-8</content_type>
|
||||
<http_response_headers>
|
||||
<Content-Language>en</Content-Language>
|
||||
<X-My-Custom-Header>43</X-My-Custom-Header>
|
||||
</http_response_headers>
|
||||
<response_content>Say Hi!</response_content>
|
||||
</handler>
|
||||
</rule>
|
||||
<defaults/>
|
||||
</http_handlers>
|
||||
```
|
||||
|
||||
`http_response_headers` could be used to set content type instead of `content_type`.
|
||||
|
||||
``` xml
|
||||
<http_handlers>
|
||||
<rule>
|
||||
<methods>GET</methods>
|
||||
<headers><XXX>xxx</XXX></headers>
|
||||
<url>/hi</url>
|
||||
<handler>
|
||||
<type>static</type>
|
||||
<status>402</status>
|
||||
<http_response_headers>
|
||||
<Content-Type>text/html; charset=UTF-8</Content-Type>
|
||||
<Content-Language>en</Content-Language>
|
||||
<X-My-Custom-Header>43</X-My-Custom-Header>
|
||||
</http_response_headers>
|
||||
<response_content>Say Hi!</response_content>
|
||||
</handler>
|
||||
</rule>
|
||||
@ -696,6 +725,9 @@ Find the content from the file send to client.
|
||||
<handler>
|
||||
<type>static</type>
|
||||
<content_type>text/html; charset=UTF-8</content_type>
|
||||
<http_response_headers>
|
||||
<ETag>737060cd8c284d8af7ad3082f209582d</ETag>
|
||||
</http_response_headers>
|
||||
<response_content>file:///absolute_path_file.html</response_content>
|
||||
</handler>
|
||||
</rule>
|
||||
@ -706,6 +738,9 @@ Find the content from the file send to client.
|
||||
<handler>
|
||||
<type>static</type>
|
||||
<content_type>text/html; charset=UTF-8</content_type>
|
||||
<http_response_headers>
|
||||
<ETag>737060cd8c284d8af7ad3082f209582d</ETag>
|
||||
</http_response_headers>
|
||||
<response_content>file://./relative_path_file.html</response_content>
|
||||
</handler>
|
||||
</rule>
|
||||
|
||||
@ -31,6 +31,56 @@ Alternatively, in order to enable the MySQL interface for an existing service:
|
||||
3. After entering the password, you will get prompted the MySQL connection string for this service
|
||||
|
||||
|
||||
## Creating multiple MySQL users in ClickHouse Cloud
|
||||
|
||||
By default, there is a built-in `mysql4<subdomain>` user, which uses the same password as the `default` one. The `<subdomain>` part is the first segment of your ClickHouse Cloud hostname. This format is necessary to work with the tools that implement secure connection, but don't provide [SNI information in their TLS handshake](https://www.cloudflare.com/learning/ssl/what-is-sni), which makes it impossible to do the internal routing without an extra hint in the username (MySQL console client is one of such tools).
|
||||
|
||||
Because of this, we _highly recommend_ following the `mysql4<subdomain>_<username>` format when creating a new user intended to be used with the MySQL interface, where `<subdomain>` is a hint to identify your Cloud service, and `<username>` is an arbitrary suffix of your choice.
|
||||
|
||||
:::tip
|
||||
For ClickHouse Cloud hostname like `foobar.us-east1.aws.clickhouse.cloud`, the `<subdomain>` part equals to `foobar`, and a custom MySQL username could look like `mysql4foobar_team1`.
|
||||
:::
|
||||
|
||||
You can create extra users to use with the MySQL interface if, for example, you need to apply extra settings.
|
||||
|
||||
1. Optional - create a [settings profile](https://clickhouse.com/docs/en/sql-reference/statements/create/settings-profile) to apply for your custom user. For example, `my_custom_profile` with an extra setting which will be applied by default when we connect with the user we create later:
|
||||
|
||||
```sql
|
||||
CREATE SETTINGS PROFILE my_custom_profile SETTINGS prefer_column_name_to_alias=1;
|
||||
```
|
||||
|
||||
`prefer_column_name_to_alias` is used just as an example, you can use other settings there.
|
||||
2. [Create a user](https://clickhouse.com/docs/en/sql-reference/statements/create/user) using the following format: `mysql4<subdomain>_<username>` ([see above](#creating-multiple-mysql-users-in-clickhouse-cloud)). The password must be in double SHA1 format. For example:
|
||||
|
||||
```sql
|
||||
CREATE USER mysql4foobar_team1 IDENTIFIED WITH double_sha1_password BY 'YourPassword42$';
|
||||
```
|
||||
|
||||
or if you want to use a custom profile for this user:
|
||||
|
||||
```sql
|
||||
CREATE USER mysql4foobar_team1 IDENTIFIED WITH double_sha1_password BY 'YourPassword42$' SETTINGS PROFILE 'my_custom_profile';
|
||||
```
|
||||
|
||||
where `my_custom_profile` is the name of the profile you created earlier.
|
||||
3. [Grant](https://clickhouse.com/docs/en/sql-reference/statements/grant) the new user the necessary permissions to interact with the desired tables or databases. For example, if you want to grant access to `system.query_log` only:
|
||||
|
||||
```sql
|
||||
GRANT SELECT ON system.query_log TO mysql4foobar_team1;
|
||||
```
|
||||
|
||||
4. Use the created user to connect to your ClickHouse Cloud service with the MySQL interface.
|
||||
|
||||
### Troubleshooting multiple MySQL users in ClickHouse Cloud
|
||||
|
||||
If you created a new MySQL user, and you see the following error while connecting via MySQL CLI client:
|
||||
|
||||
```
|
||||
ERROR 2013 (HY000): Lost connection to MySQL server at 'reading authorization packet', system error: 54
|
||||
```
|
||||
|
||||
In this case, ensure that the username follows the `mysql4<subdomain>_<username>` format, as described ([above](#creating-multiple-mysql-users-in-clickhouse-cloud)).
|
||||
|
||||
## Enabling the MySQL Interface On Self-managed ClickHouse
|
||||
|
||||
Add the [mysql_port](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-mysql_port) setting to your server's configuration file. For example, you could define the port in a new XML file in your `config.d/` [folder](../operations/configuration-files):
|
||||
|
||||
@ -59,10 +59,10 @@ For that, we need to use `jemalloc`'s tool called [jeprof](https://github.com/je
|
||||
If that’s the case, we recommend installing an [alternative implementation](https://github.com/gimli-rs/addr2line) of the tool.
|
||||
|
||||
```
|
||||
git clone https://github.com/gimli-rs/addr2line
|
||||
git clone https://github.com/gimli-rs/addr2line.git --depth=1 --branch=0.23.0
|
||||
cd addr2line
|
||||
cargo b --examples -r
|
||||
cp ./target/release/examples/addr2line path/to/current/addr2line
|
||||
cargo build --features bin --release
|
||||
cp ./target/release/addr2line path/to/current/addr2line
|
||||
```
|
||||
:::
|
||||
|
||||
|
||||
@ -591,6 +591,22 @@ Default value: 100000
|
||||
<max_part_num_to_warn>400</max_part_num_to_warn>
|
||||
```
|
||||
|
||||
## max\_table\_num\_to\_throw {#max-table-num-to-throw}
|
||||
If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.Default value: 0
|
||||
|
||||
**Example**
|
||||
```xml
|
||||
<max_table_num_to_throw>400</max_table_num_to_throw>
|
||||
```
|
||||
|
||||
## max\_database\_num\_to\_throw {#max-table-num-to-throw}
|
||||
If number of _database is greater than this value, server will throw an exception. 0 means no limitation.
|
||||
Default value: 0
|
||||
|
||||
**Example**
|
||||
```xml
|
||||
<max_database_num_to_throw>400</max_database_num_to_throw>
|
||||
```
|
||||
|
||||
## max_temporary_data_on_disk_size
|
||||
|
||||
@ -938,6 +954,38 @@ Or it can be set in hex:
|
||||
|
||||
Everything mentioned above can be applied for `aes_256_gcm_siv` (but the key must be 32 bytes long).
|
||||
|
||||
## error_log {#error_log}
|
||||
|
||||
It is disabled by default.
|
||||
|
||||
**Enabling**
|
||||
|
||||
To manually turn on error history collection [`system.error_log`](../../operations/system-tables/error_log.md), create `/etc/clickhouse-server/config.d/error_log.xml` with the following content:
|
||||
|
||||
``` xml
|
||||
<clickhouse>
|
||||
<error_log>
|
||||
<database>system</database>
|
||||
<table>error_log</table>
|
||||
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
|
||||
<collect_interval_milliseconds>1000</collect_interval_milliseconds>
|
||||
<max_size_rows>1048576</max_size_rows>
|
||||
<reserved_size_rows>8192</reserved_size_rows>
|
||||
<buffer_size_rows_flush_threshold>524288</buffer_size_rows_flush_threshold>
|
||||
<flush_on_crash>false</flush_on_crash>
|
||||
</error_log>
|
||||
</clickhouse>
|
||||
```
|
||||
|
||||
**Disabling**
|
||||
|
||||
To disable `error_log` setting, you should create the following file `/etc/clickhouse-server/config.d/disable_error_log.xml` with the following content:
|
||||
|
||||
``` xml
|
||||
<clickhouse>
|
||||
<error_log remove="1" />
|
||||
</clickhouse>
|
||||
```
|
||||
|
||||
## custom_settings_prefixes {#custom_settings_prefixes}
|
||||
|
||||
@ -1901,7 +1949,7 @@ For more information, see the MergeTreeSettings.h header file.
|
||||
|
||||
## metric_log {#metric_log}
|
||||
|
||||
It is enabled by default. If it`s not, you can do this manually.
|
||||
It is disabled by default.
|
||||
|
||||
**Enabling**
|
||||
|
||||
@ -3085,6 +3133,24 @@ Type: Bool
|
||||
|
||||
Default: 1
|
||||
|
||||
## merge_workload {#merge_workload}
|
||||
|
||||
Used to regulate how resources are utilized and shared between merges and other workloads. Specified value is used as `workload` setting value for all background merges. Can be overridden by a merge tree setting.
|
||||
|
||||
Default value: "default"
|
||||
|
||||
**See Also**
|
||||
- [Workload Scheduling](/docs/en/operations/workload-scheduling.md)
|
||||
|
||||
## mutation_workload {#mutation_workload}
|
||||
|
||||
Used to regulate how resources are utilized and shared between mutations and other workloads. Specified value is used as `workload` setting value for all background mutations. Can be overridden by a merge tree setting.
|
||||
|
||||
Default value: "default"
|
||||
|
||||
**See Also**
|
||||
- [Workload Scheduling](/docs/en/operations/workload-scheduling.md)
|
||||
|
||||
## use_legacy_mongodb_integration
|
||||
|
||||
Use the legacy MongoDB integration implementation. Deprecated.
|
||||
|
||||
@ -974,6 +974,24 @@ Default value: false
|
||||
|
||||
- [exclude_deleted_rows_for_part_size_in_merge](#exclude_deleted_rows_for_part_size_in_merge) setting
|
||||
|
||||
## merge_workload
|
||||
|
||||
Used to regulate how resources are utilized and shared between merges and other workloads. Specified value is used as `workload` setting value for background merges of this table. If not specified (empty string), then server setting `merge_workload` is used instead.
|
||||
|
||||
Default value: an empty string
|
||||
|
||||
**See Also**
|
||||
- [Workload Scheduling](/docs/en/operations/workload-scheduling.md)
|
||||
|
||||
## mutation_workload
|
||||
|
||||
Used to regulate how resources are utilized and shared between mutations and other workloads. Specified value is used as `workload` setting value for background mutations of this table. If not specified (empty string), then server setting `mutation_workload` is used instead.
|
||||
|
||||
Default value: an empty string
|
||||
|
||||
**See Also**
|
||||
- [Workload Scheduling](/docs/en/operations/workload-scheduling.md)
|
||||
|
||||
### optimize_row_order
|
||||
|
||||
Controls if the row order should be optimized during inserts to improve the compressability of the newly inserted table part.
|
||||
|
||||
@ -1706,6 +1706,43 @@ Result:
|
||||
└────────────┘
|
||||
```
|
||||
|
||||
## output_format_pretty_display_footer_column_names
|
||||
|
||||
Display column names in the footer if there are many table rows.
|
||||
|
||||
Possible values:
|
||||
|
||||
- 0 — No column names are displayed in the footer.
|
||||
- 1 — Column names are displayed in the footer if row count is greater than or equal to the threshold value set by [output_format_pretty_display_footer_column_names_min_rows](#output_format_pretty_display_footer_column_names_min_rows) (50 by default).
|
||||
|
||||
Default value: `1`.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT *, toTypeName(*) FROM (SELECT * FROM system.numbers LIMIT 1000);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─number─┬─toTypeName(number)─┐
|
||||
1. │ 0 │ UInt64 │
|
||||
2. │ 1 │ UInt64 │
|
||||
3. │ 2 │ UInt64 │
|
||||
...
|
||||
999. │ 998 │ UInt64 │
|
||||
1000. │ 999 │ UInt64 │
|
||||
└─number─┴─toTypeName(number)─┘
|
||||
```
|
||||
## output_format_pretty_display_footer_column_names_min_rows
|
||||
|
||||
Sets the minimum number of rows for which a footer with column names will be displayed if setting [output_format_pretty_display_footer_column_names](#output_format_pretty_display_footer_column_names) is enabled.
|
||||
|
||||
Default value: `50`.
|
||||
|
||||
## Template format settings {#template-format-settings}
|
||||
|
||||
### format_template_resultset {#format_template_resultset}
|
||||
|
||||
@ -1592,19 +1592,19 @@ Default value: `default`.
|
||||
|
||||
## parallel_replicas_custom_key_range_lower {#parallel_replicas_custom_key_range_lower}
|
||||
|
||||
Allows the filter type `range` to split the work evenly between replicas based on the custom range `[parallel_replicas_custom_key_range_lower, INT_MAX]`.
|
||||
Allows the filter type `range` to split the work evenly between replicas based on the custom range `[parallel_replicas_custom_key_range_lower, INT_MAX]`.
|
||||
|
||||
When used in conjuction with [parallel_replicas_custom_key_range_upper](#parallel_replicas_custom_key_range_upper), it lets the filter evenly split the work over replicas for the range `[parallel_replicas_custom_key_range_lower, parallel_replicas_custom_key_range_upper]`.
|
||||
When used in conjuction with [parallel_replicas_custom_key_range_upper](#parallel_replicas_custom_key_range_upper), it lets the filter evenly split the work over replicas for the range `[parallel_replicas_custom_key_range_lower, parallel_replicas_custom_key_range_upper]`.
|
||||
|
||||
Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing.
|
||||
Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing.
|
||||
|
||||
## parallel_replicas_custom_key_range_upper {#parallel_replicas_custom_key_range_upper}
|
||||
|
||||
Allows the filter type `range` to split the work evenly between replicas based on the custom range `[0, parallel_replicas_custom_key_range_upper]`. A value of 0 disables the upper bound, setting it the max value of the custom key expression.
|
||||
|
||||
When used in conjuction with [parallel_replicas_custom_key_range_lower](#parallel_replicas_custom_key_range_lower), it lets the filter evenly split the work over replicas for the range `[parallel_replicas_custom_key_range_lower, parallel_replicas_custom_key_range_upper]`.
|
||||
When used in conjuction with [parallel_replicas_custom_key_range_lower](#parallel_replicas_custom_key_range_lower), it lets the filter evenly split the work over replicas for the range `[parallel_replicas_custom_key_range_lower, parallel_replicas_custom_key_range_upper]`.
|
||||
|
||||
Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing.
|
||||
Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing.
|
||||
|
||||
## allow_experimental_parallel_reading_from_replicas
|
||||
|
||||
@ -3188,7 +3188,7 @@ Default value: `0`.
|
||||
|
||||
## lightweight_deletes_sync {#lightweight_deletes_sync}
|
||||
|
||||
The same as 'mutation_sync', but controls only execution of lightweight deletes.
|
||||
The same as 'mutation_sync', but controls only execution of lightweight deletes.
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -5150,7 +5150,7 @@ Allows using statistic to optimize the order of [prewhere conditions](../../sql-
|
||||
|
||||
## analyze_index_with_space_filling_curves
|
||||
|
||||
If a table has a space-filling curve in its index, e.g. `ORDER BY mortonEncode(x, y)`, and the query has conditions on its arguments, e.g. `x >= 10 AND x <= 20 AND y >= 20 AND y <= 30`, use the space-filling curve for index analysis.
|
||||
If a table has a space-filling curve in its index, e.g. `ORDER BY mortonEncode(x, y)` or `ORDER BY hilbertEncode(x, y)`, and the query has conditions on its arguments, e.g. `x >= 10 AND x <= 20 AND y >= 20 AND y <= 30`, use the space-filling curve for index analysis.
|
||||
|
||||
## query_plan_enable_optimizations {#query_plan_enable_optimizations}
|
||||
|
||||
@ -5418,6 +5418,12 @@ When set to `false` than all attempts are made with identical timeouts.
|
||||
|
||||
Default value: `true`.
|
||||
|
||||
## uniform_snowflake_conversion_functions {#uniform_snowflake_conversion_functions}
|
||||
|
||||
If set to `true`, then functions `snowflakeIDToDateTime`, `snowflakeIDToDateTime64`, `dateTimeToSnowflakeID`, and `dateTime64ToSnowflakeID` are enabled, and functions `snowflakeToDateTime`, `snowflakeToDateTime64`, `dateTimeToSnowflake`, and `dateTime64ToSnowflake` are disabled (and vice versa if set to `false`).
|
||||
|
||||
Default value: `true`
|
||||
|
||||
## allow_experimental_variant_type {#allow_experimental_variant_type}
|
||||
|
||||
Allows creation of experimental [Variant](../../sql-reference/data-types/variant.md).
|
||||
|
||||
@ -639,6 +639,10 @@ An internal metric of the low-level memory allocator (jemalloc). See https://jem
|
||||
|
||||
An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html
|
||||
|
||||
### jemalloc.prof.active
|
||||
|
||||
An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html
|
||||
|
||||
**See Also**
|
||||
|
||||
- [Monitoring](../../operations/monitoring.md) — Base concepts of ClickHouse monitoring.
|
||||
|
||||
39
docs/en/operations/system-tables/error_log.md
Normal file
39
docs/en/operations/system-tables/error_log.md
Normal file
@ -0,0 +1,39 @@
|
||||
---
|
||||
slug: /en/operations/system-tables/error_log
|
||||
---
|
||||
# error_log
|
||||
|
||||
Contains history of error values from table `system.errors`, periodically flushed to disk.
|
||||
|
||||
Columns:
|
||||
- `hostname` ([LowCardinality(String)](../../sql-reference/data-types/string.md)) — Hostname of the server executing the query.
|
||||
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date.
|
||||
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time.
|
||||
- `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — Code number of the error.
|
||||
- `error` ([LowCardinality(String)](../../sql-reference/data-types/string.md)) - Name of the error.
|
||||
- `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of times this error happened.
|
||||
- `remote` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Remote exception (i.e. received during one of the distributed queries).
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT * FROM system.error_log LIMIT 1 FORMAT Vertical;
|
||||
```
|
||||
|
||||
``` text
|
||||
Row 1:
|
||||
──────
|
||||
hostname: clickhouse.eu-central1.internal
|
||||
event_date: 2024-06-18
|
||||
event_time: 2024-06-18 07:32:39
|
||||
code: 999
|
||||
error: KEEPER_EXCEPTION
|
||||
value: 2
|
||||
remote: 0
|
||||
```
|
||||
|
||||
**See also**
|
||||
|
||||
- [error_log setting](../../operations/server-configuration-parameters/settings.md#error_log) — Enabling and disabling the setting.
|
||||
- [system.errors](../../operations/system-tables/errors.md) — Contains error codes with the number of times they have been triggered.
|
||||
- [Monitoring](../../operations/monitoring.md) — Base concepts of ClickHouse monitoring.
|
||||
@ -113,6 +113,8 @@ Columns:
|
||||
- `used_functions` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `functions`, which were used during query execution.
|
||||
- `used_storages` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `storages`, which were used during query execution.
|
||||
- `used_table_functions` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `table functions`, which were used during query execution.
|
||||
- `used_privileges` ([Array(String)](../../sql-reference/data-types/array.md)) - Privileges which were successfully checked during query execution.
|
||||
- `missing_privileges` ([Array(String)](../../sql-reference/data-types/array.md)) - Privileges that are missing during query execution.
|
||||
- `query_cache_usage` ([Enum8](../../sql-reference/data-types/enum.md)) — Usage of the [query cache](../query-cache.md) during query execution. Values:
|
||||
- `'Unknown'` = Status unknown.
|
||||
- `'None'` = The query result was neither written into nor read from the query cache.
|
||||
@ -194,6 +196,8 @@ used_formats: []
|
||||
used_functions: []
|
||||
used_storages: []
|
||||
used_table_functions: []
|
||||
used_privileges: []
|
||||
missing_privileges: []
|
||||
query_cache_usage: None
|
||||
```
|
||||
|
||||
|
||||
@ -47,6 +47,8 @@ Example:
|
||||
|
||||
Queries can be marked with setting `workload` to distinguish different workloads. If `workload` is not set, than value "default" is used. Note that you are able to specify the other value using settings profiles. Setting constraints can be used to make `workload` constant if you want all queries from the user to be marked with fixed value of `workload` setting.
|
||||
|
||||
It is possible to assign a `workload` setting for background activities. Merges and mutations are using `merge_workload` and `mutation_workload` server settings correspondingly. These values can also be overridden for specific tables using `merge_workload` and `mutation_workload` merge tree settings
|
||||
|
||||
Let's consider an example of a system with two different workloads: "production" and "development".
|
||||
|
||||
```sql
|
||||
@ -151,6 +153,9 @@ Example:
|
||||
</clickhouse>
|
||||
```
|
||||
|
||||
|
||||
## See also
|
||||
- [system.scheduler](/docs/en/operations/system-tables/scheduler.md)
|
||||
- [merge_workload](/docs/en/operations/settings/merge-tree-settings.md#merge_workload) merge tree setting
|
||||
- [merge_workload](/docs/en/operations/server-configuration-parameters/settings.md#merge_workload) global server setting
|
||||
- [mutation_workload](/docs/en/operations/settings/merge-tree-settings.md#mutation_workload) merge tree setting
|
||||
- [mutation_workload](/docs/en/operations/server-configuration-parameters/settings.md#mutation_workload) global server setting
|
||||
|
||||
@ -25,7 +25,7 @@ stddevPop(x)
|
||||
|
||||
**Returned value**
|
||||
|
||||
Square root of standard deviation of `x`. [Float64](../../data-types/float.md).
|
||||
- Square root of standard deviation of `x`. [Float64](../../data-types/float.md).
|
||||
|
||||
|
||||
**Example**
|
||||
|
||||
@ -4,30 +4,25 @@ slug: "/en/sql-reference/aggregate-functions/reference/varpop"
|
||||
sidebar_position: 32
|
||||
---
|
||||
|
||||
This page covers the `varPop` and `varPopStable` functions available in ClickHouse.
|
||||
|
||||
## varPop
|
||||
|
||||
Calculates the population covariance between two data columns. The population covariance measures the degree to which two variables vary together. Calculates the amount `Σ((x - x̅)^2) / n`, where `n` is the sample size and `x̅`is the average value of `x`.
|
||||
Calculates the population variance.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
covarPop(x, y)
|
||||
varPop(x)
|
||||
```
|
||||
|
||||
Alias: `VAR_POP`.
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `x`: The first data column. [Numeric](../../../native-protocol/columns.md)
|
||||
- `y`: The second data column. [Numeric](../../../native-protocol/columns.md)
|
||||
- `x`: Population of values to find the population variance of. [(U)Int*](../../data-types/int-uint.md), [Float*](../../data-types/float.md), [Decimal*](../../data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
Returns an integer of type `Float64`.
|
||||
|
||||
**Implementation details**
|
||||
|
||||
This function uses a numerically unstable algorithm. If you need numerical stability in calculations, use the slower but more stable [`varPopStable`](#varpopstable) function.
|
||||
- Returns the population variance of `x`. [`Float64`](../../data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
@ -37,69 +32,21 @@ Query:
|
||||
DROP TABLE IF EXISTS test_data;
|
||||
CREATE TABLE test_data
|
||||
(
|
||||
x Int32,
|
||||
y Int32
|
||||
x UInt8,
|
||||
)
|
||||
ENGINE = Memory;
|
||||
|
||||
INSERT INTO test_data VALUES (1, 2), (2, 3), (3, 5), (4, 6), (5, 8);
|
||||
INSERT INTO test_data VALUES (3), (3), (3), (4), (4), (5), (5), (7), (11), (15);
|
||||
|
||||
SELECT
|
||||
covarPop(x, y) AS covar_pop
|
||||
varPop(x) AS var_pop
|
||||
FROM test_data;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
3
|
||||
```
|
||||
|
||||
## varPopStable
|
||||
|
||||
Calculates population covariance between two data columns using a stable, numerically accurate method to calculate the variance. This function is designed to provide reliable results even with large datasets or values that might cause numerical instability in other implementations.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
covarPopStable(x, y)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `x`: The first data column. [String literal](../../syntax#syntax-string-literal)
|
||||
- `y`: The second data column. [Expression](../../syntax#syntax-expressions)
|
||||
|
||||
**Returned value**
|
||||
|
||||
Returns an integer of type `Float64`.
|
||||
|
||||
**Implementation details**
|
||||
|
||||
Unlike [`varPop`](#varpop), this function uses a stable, numerically accurate algorithm to calculate the population variance to avoid issues like catastrophic cancellation or loss of precision. This function also handles `NaN` and `Inf` values correctly, excluding them from calculations.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
DROP TABLE IF EXISTS test_data;
|
||||
CREATE TABLE test_data
|
||||
(
|
||||
x Int32,
|
||||
y Int32
|
||||
)
|
||||
ENGINE = Memory;
|
||||
|
||||
INSERT INTO test_data VALUES (1, 2), (2, 9), (9, 5), (4, 6), (5, 8);
|
||||
|
||||
SELECT
|
||||
covarPopStable(x, y) AS covar_pop_stable
|
||||
FROM test_data;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
0.5999999999999999
|
||||
┌─var_pop─┐
|
||||
│ 14.4 │
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
@ -0,0 +1,52 @@
|
||||
---
|
||||
title: "varPopStable"
|
||||
slug: "/en/sql-reference/aggregate-functions/reference/varpopstable"
|
||||
sidebar_position: 32
|
||||
---
|
||||
|
||||
## varPopStable
|
||||
|
||||
Returns the population variance. Unlike [`varPop`](../reference/varpop.md), this function uses a [numerically stable](https://en.wikipedia.org/wiki/Numerical_stability) algorithm. It works slower but provides a lower computational error.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
varPopStable(x)
|
||||
```
|
||||
|
||||
Alias: `VAR_POP_STABLE`.
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `x`: Population of values to find the population variance of. [(U)Int*](../../data-types/int-uint.md), [Float*](../../data-types/float.md), [Decimal*](../../data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Returns the population variance of `x`. [Float64](../../data-types/float.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
DROP TABLE IF EXISTS test_data;
|
||||
CREATE TABLE test_data
|
||||
(
|
||||
x UInt8,
|
||||
)
|
||||
ENGINE = Memory;
|
||||
|
||||
INSERT INTO test_data VALUES (3),(3),(3),(4),(4),(5),(5),(7),(11),(15);
|
||||
|
||||
SELECT
|
||||
varPopStable(x) AS var_pop_stable
|
||||
FROM test_data;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─var_pop_stable─┐
|
||||
│ 14.4 │
|
||||
└────────────────┘
|
||||
```
|
||||
@ -4,8 +4,6 @@ slug: /en/sql-reference/aggregate-functions/reference/varsamp
|
||||
sidebar_position: 33
|
||||
---
|
||||
|
||||
This page contains information on the `varSamp` and `varSampStable` ClickHouse functions.
|
||||
|
||||
## varSamp
|
||||
|
||||
Calculate the sample variance of a data set.
|
||||
@ -13,24 +11,27 @@ Calculate the sample variance of a data set.
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
varSamp(expr)
|
||||
varSamp(x)
|
||||
```
|
||||
|
||||
Alias: `VAR_SAMP`.
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `expr`: An expression representing the data set for which you want to calculate the sample variance. [Expression](../../syntax#syntax-expressions)
|
||||
- `x`: The population for which you want to calculate the sample variance. [(U)Int*](../../data-types/int-uint.md), [Float*](../../data-types/float.md), [Decimal*](../../data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
Returns a Float64 value representing the sample variance of the input data set.
|
||||
|
||||
- Returns the sample variance of the input data set `x`. [Float64](../../data-types/float.md).
|
||||
|
||||
**Implementation details**
|
||||
|
||||
The `varSamp()` function calculates the sample variance using the following formula:
|
||||
The `varSamp` function calculates the sample variance using the following formula:
|
||||
|
||||
```plaintext
|
||||
∑(x - mean(x))^2 / (n - 1)
|
||||
```
|
||||
$$
|
||||
\sum\frac{(x - \text{mean}(x))^2}{(n - 1)}
|
||||
$$
|
||||
|
||||
Where:
|
||||
|
||||
@ -38,91 +39,29 @@ Where:
|
||||
- `mean(x)` is the arithmetic mean of the data set.
|
||||
- `n` is the number of data points in the data set.
|
||||
|
||||
The function assumes that the input data set represents a sample from a larger population. If you want to calculate the variance of the entire population (when you have the complete data set), you should use the [`varPop()` function](./varpop#varpop) instead.
|
||||
|
||||
This function uses a numerically unstable algorithm. If you need numerical stability in calculations, use the slower but more stable [`varSampStable`](#varsampstable) function.
|
||||
The function assumes that the input data set represents a sample from a larger population. If you want to calculate the variance of the entire population (when you have the complete data set), you should use [`varPop`](../reference/varpop.md) instead.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
CREATE TABLE example_table
|
||||
DROP TABLE IF EXISTS test_data;
|
||||
CREATE TABLE test_data
|
||||
(
|
||||
id UInt64,
|
||||
value Float64
|
||||
x Float64
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY id;
|
||||
ENGINE = Memory;
|
||||
|
||||
INSERT INTO example_table VALUES (1, 10.5), (2, 12.3), (3, 9.8), (4, 11.2), (5, 10.7);
|
||||
INSERT INTO test_data VALUES (10.5), (12.3), (9.8), (11.2), (10.7);
|
||||
|
||||
SELECT varSamp(value) FROM example_table;
|
||||
SELECT round(varSamp(x),3) AS var_samp FROM test_data;
|
||||
```
|
||||
|
||||
Response:
|
||||
|
||||
```response
|
||||
0.8650000000000091
|
||||
┌─var_samp─┐
|
||||
│ 0.865 │
|
||||
└──────────┘
|
||||
```
|
||||
|
||||
## varSampStable
|
||||
|
||||
Calculate the sample variance of a data set using a numerically stable algorithm.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
varSampStable(expr)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `expr`: An expression representing the data set for which you want to calculate the sample variance. [Expression](../../syntax#syntax-expressions)
|
||||
|
||||
**Returned value**
|
||||
|
||||
The `varSampStable` function returns a Float64 value representing the sample variance of the input data set.
|
||||
|
||||
**Implementation details**
|
||||
|
||||
The `varSampStable` function calculates the sample variance using the same formula as the [`varSamp`](#varsamp) function:
|
||||
|
||||
```plaintext
|
||||
∑(x - mean(x))^2 / (n - 1)
|
||||
```
|
||||
|
||||
Where:
|
||||
- `x` is each individual data point in the data set.
|
||||
- `mean(x)` is the arithmetic mean of the data set.
|
||||
- `n` is the number of data points in the data set.
|
||||
|
||||
The difference between `varSampStable` and `varSamp` is that `varSampStable` is designed to provide a more deterministic and stable result when dealing with floating-point arithmetic. It uses an algorithm that minimizes the accumulation of rounding errors, which can be particularly important when dealing with large data sets or data with a wide range of values.
|
||||
|
||||
Like `varSamp`, the `varSampStable` function assumes that the input data set represents a sample from a larger population. If you want to calculate the variance of the entire population (when you have the complete data set), you should use the [`varPopStable`](./varpop#varpopstable) function instead.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
CREATE TABLE example_table
|
||||
(
|
||||
id UInt64,
|
||||
value Float64
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY id;
|
||||
|
||||
INSERT INTO example_table VALUES (1, 10.5), (2, 12.3), (3, 9.8), (4, 11.2), (5, 10.7);
|
||||
|
||||
SELECT varSampStable(value) FROM example_table;
|
||||
```
|
||||
|
||||
Response:
|
||||
|
||||
```response
|
||||
0.865
|
||||
```
|
||||
|
||||
This query calculates the sample variance of the `value` column in the `example_table` using the `varSampStable()` function. The result shows that the sample variance of the values `[10.5, 12.3, 9.8, 11.2, 10.7]` is approximately 0.865, which may differ slightly from the result of `varSamp` due to the more precise handling of floating-point arithmetic.
|
||||
|
||||
@ -0,0 +1,63 @@
|
||||
---
|
||||
title: "varSampStable"
|
||||
slug: /en/sql-reference/aggregate-functions/reference/varsampstable
|
||||
sidebar_position: 33
|
||||
---
|
||||
|
||||
## varSampStable
|
||||
|
||||
Calculate the sample variance of a data set. Unlike [`varSamp`](../reference/varsamp.md), this function uses a numerically stable algorithm. It works slower but provides a lower computational error.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
varSampStable(x)
|
||||
```
|
||||
|
||||
Alias: `VAR_SAMP_STABLE`
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `x`: The population for which you want to calculate the sample variance. [(U)Int*](../../data-types/int-uint.md), [Float*](../../data-types/float.md), [Decimal*](../../data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Returns the sample variance of the input data set. [Float64](../../data-types/float.md).
|
||||
|
||||
**Implementation details**
|
||||
|
||||
The `varSampStable` function calculates the sample variance using the same formula as the [`varSamp`](../reference/varsamp.md):
|
||||
|
||||
$$
|
||||
\sum\frac{(x - \text{mean}(x))^2}{(n - 1)}
|
||||
$$
|
||||
|
||||
Where:
|
||||
- `x` is each individual data point in the data set.
|
||||
- `mean(x)` is the arithmetic mean of the data set.
|
||||
- `n` is the number of data points in the data set.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
DROP TABLE IF EXISTS test_data;
|
||||
CREATE TABLE test_data
|
||||
(
|
||||
x Float64
|
||||
)
|
||||
ENGINE = Memory;
|
||||
|
||||
INSERT INTO test_data VALUES (10.5), (12.3), (9.8), (11.2), (10.7);
|
||||
|
||||
SELECT round(varSampStable(x),3) AS var_samp_stable FROM test_data;
|
||||
```
|
||||
|
||||
Response:
|
||||
|
||||
```response
|
||||
┌─var_samp_stable─┐
|
||||
│ 0.865 │
|
||||
└─────────────────┘
|
||||
```
|
||||
@ -137,7 +137,7 @@ If the time transition (due to daylight saving time or for other reasons) was pe
|
||||
|
||||
Non-monotonic calendar dates. For example, in Happy Valley - Goose Bay, the time was transitioned one hour backwards at 00:01:00 7 Nov 2010 (one minute after midnight). So after 6th Nov has ended, people observed a whole one minute of 7th Nov, then time was changed back to 23:01 6th Nov and after another 59 minutes the 7th Nov started again. ClickHouse does not (yet) support this kind of fun. During these days the results of time processing functions may be slightly incorrect.
|
||||
|
||||
Similar issue exists for Casey Antarctic station in year 2010. They changed time three hours back at 5 Mar, 02:00. If you are working in antarctic station, please don't afraid to use ClickHouse. Just make sure you set timezone to UTC or be aware of inaccuracies.
|
||||
Similar issue exists for Casey Antarctic station in year 2010. They changed time three hours back at 5 Mar, 02:00. If you are working in antarctic station, please don't be afraid to use ClickHouse. Just make sure you set timezone to UTC or be aware of inaccuracies.
|
||||
|
||||
Time shifts for multiple days. Some pacific islands changed their timezone offset from UTC+14 to UTC-12. That's alright but some inaccuracies may present if you do calculations with their timezone for historical time points at the days of conversion.
|
||||
|
||||
|
||||
@ -7,33 +7,43 @@ sidebar_label: Float32, Float64
|
||||
# Float32, Float64
|
||||
|
||||
:::note
|
||||
If you need accurate calculations, in particular if you work with financial or business data requiring a high precision you should consider using Decimal instead. Floats might lead to inaccurate results as illustrated below:
|
||||
If you need accurate calculations, in particular if you work with financial or business data requiring a high precision, you should consider using [Decimal](../data-types/decimal.md) instead.
|
||||
|
||||
```
|
||||
[Floating Point Numbers](https://en.wikipedia.org/wiki/IEEE_754) might lead to inaccurate results as illustrated below:
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS float_vs_decimal
|
||||
(
|
||||
my_float Float64,
|
||||
my_decimal Decimal64(3)
|
||||
)Engine=MergeTree ORDER BY tuple()
|
||||
|
||||
INSERT INTO float_vs_decimal SELECT round(randCanonical(), 3) AS res, res FROM system.numbers LIMIT 1000000; # Generate 1 000 000 random number with 2 decimal places and store them as a float and as a decimal
|
||||
)
|
||||
Engine=MergeTree
|
||||
ORDER BY tuple();
|
||||
|
||||
# Generate 1 000 000 random numbers with 2 decimal places and store them as a float and as a decimal
|
||||
INSERT INTO float_vs_decimal SELECT round(randCanonical(), 3) AS res, res FROM system.numbers LIMIT 1000000;
|
||||
```
|
||||
```
|
||||
SELECT sum(my_float), sum(my_decimal) FROM float_vs_decimal;
|
||||
> 500279.56300000014 500279.563
|
||||
|
||||
┌──────sum(my_float)─┬─sum(my_decimal)─┐
|
||||
│ 499693.60500000004 │ 499693.605 │
|
||||
└────────────────────┴─────────────────┘
|
||||
|
||||
SELECT sumKahan(my_float), sumKahan(my_decimal) FROM float_vs_decimal;
|
||||
> 500279.563 500279.563
|
||||
|
||||
┌─sumKahan(my_float)─┬─sumKahan(my_decimal)─┐
|
||||
│ 499693.605 │ 499693.605 │
|
||||
└────────────────────┴──────────────────────┘
|
||||
```
|
||||
:::
|
||||
|
||||
[Floating point numbers](https://en.wikipedia.org/wiki/IEEE_754).
|
||||
|
||||
Types are equivalent to types of C:
|
||||
The equivalent types in ClickHouse and in C are given below:
|
||||
|
||||
- `Float32` — `float`.
|
||||
- `Float64` — `double`.
|
||||
|
||||
Aliases:
|
||||
Float types in ClickHouse have the following aliases:
|
||||
|
||||
- `Float32` — `FLOAT`, `REAL`, `SINGLE`.
|
||||
- `Float64` — `DOUBLE`, `DOUBLE PRECISION`.
|
||||
|
||||
@ -2178,6 +2178,32 @@ Result:
|
||||
|
||||
Alias: levenshteinDistance
|
||||
|
||||
## editDistanceUTF8
|
||||
|
||||
Calculates the [edit distance](https://en.wikipedia.org/wiki/Edit_distance) between two UTF8 strings.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
editDistanceUTF8(string1, string2)
|
||||
```
|
||||
|
||||
**Examples**
|
||||
|
||||
``` sql
|
||||
SELECT editDistanceUTF8('我是谁', '我是我');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─editDistanceUTF8('我是谁', '我是我')──┐
|
||||
│ 1 │
|
||||
└─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Alias: levenshteinDistanceUTF8
|
||||
|
||||
## damerauLevenshteinDistance
|
||||
|
||||
Calculates the [Damerau-Levenshtein distance](https://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance) between two byte strings.
|
||||
|
||||
@ -543,12 +543,17 @@ serverUUID()
|
||||
|
||||
Generates a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID).
|
||||
|
||||
The generated Snowflake ID contains the current Unix timestamp in milliseconds 41 (+ 1 top zero bit) bits, followed by machine id (10 bits), a counter (12 bits) to distinguish IDs within a millisecond.
|
||||
The generated Snowflake ID contains the current Unix timestamp in milliseconds (41 + 1 top zero bits), followed by a machine id (10 bits), and a counter (12 bits) to distinguish IDs within a millisecond.
|
||||
For any given timestamp (unix_ts_ms), the counter starts at 0 and is incremented by 1 for each new Snowflake ID until the timestamp changes.
|
||||
In case the counter overflows, the timestamp field is incremented by 1 and the counter is reset to 0.
|
||||
|
||||
Function `generateSnowflakeID` guarantees that the counter field within a timestamp increments monotonically across all function invocations in concurrently running threads and queries.
|
||||
|
||||
:::note
|
||||
The generated Snowflake IDs are based on the UNIX epoch 1970-01-01.
|
||||
While no standard or recommendation exists for the epoch of Snowflake IDs, implementations in other systems may use a different epoch, e.g. Twitter/X (2010-11-04) or Mastodon (2015-01-01).
|
||||
:::
|
||||
|
||||
```
|
||||
0 1 2 3
|
||||
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
|
||||
@ -605,6 +610,11 @@ SELECT generateSnowflakeID(1), generateSnowflakeID(2);
|
||||
|
||||
## snowflakeToDateTime
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Extracts the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) in [DateTime](../data-types/datetime.md) format.
|
||||
|
||||
**Syntax**
|
||||
@ -641,6 +651,11 @@ Result:
|
||||
|
||||
## snowflakeToDateTime64
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Extracts the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) in [DateTime64](../data-types/datetime64.md) format.
|
||||
|
||||
**Syntax**
|
||||
@ -677,6 +692,11 @@ Result:
|
||||
|
||||
## dateTimeToSnowflake
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Converts a [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
@ -711,6 +731,11 @@ Result:
|
||||
|
||||
## dateTime64ToSnowflake
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Convert a [DateTime64](../data-types/datetime64.md) to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
@ -743,6 +768,148 @@ Result:
|
||||
└─────────────────────────────┘
|
||||
```
|
||||
|
||||
## snowflakeIDToDateTime
|
||||
|
||||
Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime](../data-types/datetime.md).
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
snowflakeIDToDateTime(value[, epoch[, time_zone]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Snowflake ID. [UInt64](../data-types/int-uint.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
- `time_zone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The timestamp component of `value` as a [DateTime](../data-types/datetime.md) value.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT snowflakeIDToDateTime(7204436857747984384) AS res
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌─────────────────res─┐
|
||||
│ 2024-06-06 10:59:58 │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## snowflakeIDToDateTime64
|
||||
|
||||
Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime64](../data-types/datetime64.md).
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
snowflakeIDToDateTime64(value[, epoch[, time_zone]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Snowflake ID. [UInt64](../data-types/int-uint.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
- `time_zone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The timestamp component of `value` as a [DateTime64](../data-types/datetime64.md) with scale = 3, i.e. millisecond precision.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT snowflakeIDToDateTime64(7204436857747984384) AS res
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌─────────────────res─┐
|
||||
│ 2024-06-06 10:59:58 │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## dateTimeToSnowflakeID
|
||||
|
||||
Converts a [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
dateTimeToSnowflakeID(value[, epoch])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Date with time. [DateTime](../data-types/datetime.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai') AS dt, dateTimeToSnowflakeID(dt) AS res;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌──────────────────dt─┬─────────────────res─┐
|
||||
│ 2021-08-15 18:57:56 │ 6832626392367104000 │
|
||||
└─────────────────────┴─────────────────────┘
|
||||
```
|
||||
|
||||
## dateTime64ToSnowflakeID
|
||||
|
||||
Convert a [DateTime64](../data-types/datetime64.md) to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
dateTime64ToSnowflakeID(value[, epoch])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Date with time. [DateTime64](../data-types/datetime64.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT toDateTime('2021-08-15 18:57:56.493', 3, 'Asia/Shanghai') AS dt, dateTime64ToSnowflakeID(dt) AS res;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌──────────────────────dt─┬─────────────────res─┐
|
||||
│ 2021-08-15 18:57:56.493 │ 6832626394434895872 │
|
||||
└─────────────────────────┴─────────────────────┘
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [dictGetUUID](../functions/ext-dict-functions.md#ext_dict_functions-other)
|
||||
|
||||
@ -414,6 +414,8 @@ $ curl -v 'http://localhost:8123/predefined_query'
|
||||
|
||||
- `content_type` — используется со всеми типами, возвращает [content-type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type).
|
||||
|
||||
- `http_response_headers` — используется со всеми типами чтобы добавить кастомные хедеры в ответ. Может использоваться в том числе для задания хедера `Content-Type` вместо `content_type`.
|
||||
|
||||
- `response_content` — используется с типом`static`, содержимое ответа, отправленное клиенту, при использовании префикса ‘file://’ or ‘config://’, находит содержимое из файла или конфигурации, отправленного клиенту.
|
||||
|
||||
Далее приведены методы настройки для различных типов.
|
||||
@ -509,6 +511,33 @@ max_final_threads 2
|
||||
<type>static</type>
|
||||
<status>402</status>
|
||||
<content_type>text/html; charset=UTF-8</content_type>
|
||||
<http_response_headers>
|
||||
<Content-Language>en</Content-Language>
|
||||
<X-My-Custom-Header>43</X-My-Custom-Header>
|
||||
</http_response_headers>
|
||||
<response_content>Say Hi!</response_content>
|
||||
</handler>
|
||||
</rule>
|
||||
<defaults/>
|
||||
</http_handlers>
|
||||
```
|
||||
|
||||
`http_response_headers` так же может использоваться для определения `Content-Type` вместо `content_type`.
|
||||
|
||||
``` xml
|
||||
<http_handlers>
|
||||
<rule>
|
||||
<methods>GET</methods>
|
||||
<headers><XXX>xxx</XXX></headers>
|
||||
<url>/hi</url>
|
||||
<handler>
|
||||
<type>static</type>
|
||||
<status>402</status>
|
||||
<http_response_headers>
|
||||
<Content-Type>text/html; charset=UTF-8</Content-Type>
|
||||
<Content-Language>en</Content-Language>
|
||||
<X-My-Custom-Header>43</X-My-Custom-Header>
|
||||
</http_response_headers>
|
||||
<response_content>Say Hi!</response_content>
|
||||
</handler>
|
||||
</rule>
|
||||
@ -589,6 +618,9 @@ $ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler'
|
||||
<handler>
|
||||
<type>static</type>
|
||||
<content_type>text/html; charset=UTF-8</content_type>
|
||||
<http_response_headers>
|
||||
<ETag>737060cd8c284d8af7ad3082f209582d</ETag>
|
||||
</http_response_headers>
|
||||
<response_content>file:///absolute_path_file.html</response_content>
|
||||
</handler>
|
||||
</rule>
|
||||
@ -599,6 +631,9 @@ $ curl -v -H 'XXX:xxx' 'http://localhost:8123/get_config_static_handler'
|
||||
<handler>
|
||||
<type>static</type>
|
||||
<content_type>text/html; charset=UTF-8</content_type>
|
||||
<http_response_headers>
|
||||
<ETag>737060cd8c284d8af7ad3082f209582d</ETag>
|
||||
</http_response_headers>
|
||||
<response_content>file://./relative_path_file.html</response_content>
|
||||
</handler>
|
||||
</rule>
|
||||
|
||||
@ -368,7 +368,7 @@ int KeeperClient::main(const std::vector<String> & /* args */)
|
||||
return 0;
|
||||
}
|
||||
|
||||
DB::ConfigProcessor config_processor(config().getString("config-file", "config.xml"));
|
||||
ConfigProcessor config_processor(config().getString("config-file", "config.xml"));
|
||||
|
||||
/// This will handle a situation when clickhouse is running on the embedded config, but config.d folder is also present.
|
||||
ConfigProcessor::registerEmbeddedConfig("config.xml", "<clickhouse/>");
|
||||
|
||||
@ -12,8 +12,7 @@ bool parseKeeperArg(IParser::Pos & pos, Expected & expected, String & result)
|
||||
if (!parseIdentifierOrStringLiteral(pos, expected, result))
|
||||
return false;
|
||||
}
|
||||
|
||||
while (pos->type != TokenType::Whitespace && pos->type != TokenType::EndOfStream && pos->type != TokenType::Semicolon)
|
||||
else if (pos->type == TokenType::Number)
|
||||
{
|
||||
result.append(pos->begin, pos->end);
|
||||
++pos;
|
||||
@ -40,8 +39,8 @@ bool KeeperParser::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
for (const auto & pair : KeeperClient::commands)
|
||||
expected.add(pos, pair.first.data());
|
||||
|
||||
for (const auto & flwc : four_letter_word_commands)
|
||||
expected.add(pos, flwc.data());
|
||||
for (const auto & four_letter_word_command : four_letter_word_commands)
|
||||
expected.add(pos, four_letter_word_command.data());
|
||||
|
||||
if (pos->type != TokenType::BareWord)
|
||||
return false;
|
||||
|
||||
@ -11,7 +11,6 @@ namespace DB
|
||||
{
|
||||
|
||||
bool parseKeeperArg(IParser::Pos & pos, Expected & expected, String & result);
|
||||
|
||||
bool parseKeeperPath(IParser::Pos & pos, Expected & expected, String & path);
|
||||
|
||||
|
||||
|
||||
@ -732,11 +732,8 @@ void LocalServer::processConfig()
|
||||
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
|
||||
}
|
||||
|
||||
server_display_name = config().getString("display_name", getFQDNOrHostName());
|
||||
prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name.default", "{display_name} :) ");
|
||||
std::map<String, String> prompt_substitutions{{"display_name", server_display_name}};
|
||||
for (const auto & [key, value] : prompt_substitutions)
|
||||
boost::replace_all(prompt_by_server_display_name, "{" + key + "}", value);
|
||||
server_display_name = config().getString("display_name", "");
|
||||
prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name.default", ":) ");
|
||||
|
||||
global_context->setQueryKindInitial();
|
||||
global_context->setQueryKind(query_kind);
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
#include <Poco/Net/NetException.h>
|
||||
#include <Poco/Util/HelpFormatter.h>
|
||||
#include <Poco/Environment.h>
|
||||
#include <Poco/Config.h>
|
||||
#include <Common/scope_guard_safe.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <base/phdr_cache.h>
|
||||
@ -721,11 +722,6 @@ try
|
||||
CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::getVersionRevision());
|
||||
CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
|
||||
|
||||
Poco::ThreadPool server_pool(3, server_settings.max_connections);
|
||||
std::mutex servers_lock;
|
||||
std::vector<ProtocolServerAdapter> servers;
|
||||
std::vector<ProtocolServerAdapter> servers_to_start_before_tables;
|
||||
|
||||
/** Context contains all that query execution is dependent:
|
||||
* settings, available functions, data types, aggregate functions, databases, ...
|
||||
*/
|
||||
@ -823,6 +819,18 @@ try
|
||||
total_memory_tracker.setSampleMaxAllocationSize(server_settings.total_memory_profiler_sample_max_allocation_size);
|
||||
}
|
||||
|
||||
Poco::ThreadPool server_pool(
|
||||
/* minCapacity */3,
|
||||
/* maxCapacity */server_settings.max_connections,
|
||||
/* idleTime */60,
|
||||
/* stackSize */POCO_THREAD_STACK_SIZE,
|
||||
server_settings.global_profiler_real_time_period_ns,
|
||||
server_settings.global_profiler_cpu_time_period_ns);
|
||||
|
||||
std::mutex servers_lock;
|
||||
std::vector<ProtocolServerAdapter> servers;
|
||||
std::vector<ProtocolServerAdapter> servers_to_start_before_tables;
|
||||
|
||||
/// Wait for all threads to avoid possible use-after-free (for example logging objects can be already destroyed).
|
||||
SCOPE_EXIT({
|
||||
Stopwatch watch;
|
||||
@ -1609,6 +1617,10 @@ try
|
||||
0, // We don't need any threads one all the parts will be deleted
|
||||
new_server_settings.max_parts_cleaning_thread_pool_size);
|
||||
|
||||
|
||||
global_context->setMergeWorkload(new_server_settings.merge_workload);
|
||||
global_context->setMutationWorkload(new_server_settings.mutation_workload);
|
||||
|
||||
if (config->has("resources"))
|
||||
{
|
||||
global_context->getResourceManager()->updateConfiguration(*config);
|
||||
|
||||
@ -371,7 +371,7 @@
|
||||
<!-- Enables asynchronous loading of databases and tables to speedup server startup.
|
||||
Queries to not yet loaded entity will be blocked until load is finished.
|
||||
-->
|
||||
<!-- <async_load_databases>true</async_load_databases> -->
|
||||
<async_load_databases>true</async_load_databases>
|
||||
|
||||
<!-- On memory constrained environments you may have to set this to value larger than 1.
|
||||
-->
|
||||
@ -1155,6 +1155,18 @@
|
||||
<flush_on_crash>false</flush_on_crash>
|
||||
</metric_log>
|
||||
|
||||
<!-- Error log contains rows with current values of errors collected with "collect_interval_milliseconds" interval. -->
|
||||
<error_log>
|
||||
<database>system</database>
|
||||
<table>error_log</table>
|
||||
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
|
||||
<max_size_rows>1048576</max_size_rows>
|
||||
<reserved_size_rows>8192</reserved_size_rows>
|
||||
<buffer_size_rows_flush_threshold>524288</buffer_size_rows_flush_threshold>
|
||||
<collect_interval_milliseconds>1000</collect_interval_milliseconds>
|
||||
<flush_on_crash>false</flush_on_crash>
|
||||
</error_log>
|
||||
|
||||
<!--
|
||||
Asynchronous metric log contains values of metrics from
|
||||
system.asynchronous_metrics.
|
||||
@ -1396,6 +1408,14 @@
|
||||
<!-- <host_name>replica</host_name> -->
|
||||
</distributed_ddl>
|
||||
|
||||
<!-- Used to regulate how resources are utilized and shared between merges, mutations and other workloads.
|
||||
Specified value is used as `workload` setting value for background merge or mutation.
|
||||
-->
|
||||
<!--
|
||||
<merge_workload>merges_and_mutations</merge_workload>
|
||||
<mutation_workload>merges_and_mutations</mutation_workload>
|
||||
-->
|
||||
|
||||
<!-- Settings to fine-tune MergeTree tables. See documentation in source code, in MergeTreeSettings.h -->
|
||||
<!--
|
||||
<merge_tree>
|
||||
|
||||
@ -726,6 +726,13 @@ metric_log:
|
||||
flush_interval_milliseconds: 7500
|
||||
collect_interval_milliseconds: 1000
|
||||
|
||||
# Error log contains rows with current values of errors collected with "collect_interval_milliseconds" interval.
|
||||
error_log:
|
||||
database: system
|
||||
table: error_log
|
||||
flush_interval_milliseconds: 7500
|
||||
collect_interval_milliseconds: 1000
|
||||
|
||||
# Asynchronous metric log contains values of metrics from
|
||||
# system.asynchronous_metrics.
|
||||
asynchronous_metric_log:
|
||||
|
||||
@ -261,7 +261,24 @@ AccessControl::AccessControl()
|
||||
}
|
||||
|
||||
|
||||
AccessControl::~AccessControl() = default;
|
||||
AccessControl::~AccessControl()
|
||||
{
|
||||
try
|
||||
{
|
||||
AccessControl::shutdown();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void AccessControl::shutdown()
|
||||
{
|
||||
MultipleAccessStorage::shutdown();
|
||||
removeAllStorages();
|
||||
}
|
||||
|
||||
|
||||
void AccessControl::setUpFromMainConfig(const Poco::Util::AbstractConfiguration & config_, const String & config_path_,
|
||||
|
||||
@ -53,6 +53,9 @@ public:
|
||||
AccessControl();
|
||||
~AccessControl() override;
|
||||
|
||||
/// Shutdown the access control and stops all background activity.
|
||||
void shutdown() override;
|
||||
|
||||
/// Initializes access storage (user directories).
|
||||
void setUpFromMainConfig(const Poco::Util::AbstractConfiguration & config_, const String & config_path_,
|
||||
const zkutil::GetZooKeeper & get_zookeeper_function_);
|
||||
|
||||
@ -31,6 +31,7 @@ namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int AUTHENTICATION_FAILED;
|
||||
extern const int SUPPORT_IS_DISABLED;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int LOGICAL_ERROR;
|
||||
@ -90,8 +91,10 @@ bool AuthenticationData::Util::checkPasswordBcrypt(std::string_view password [[m
|
||||
{
|
||||
#if USE_BCRYPT
|
||||
int ret = bcrypt_checkpw(password.data(), reinterpret_cast<const char *>(password_bcrypt.data()));
|
||||
/// Before 24.6 we didn't validate hashes on creation, so it could be that the stored hash is invalid
|
||||
/// and it could not be decoded by the library
|
||||
if (ret == -1)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "BCrypt library failed: bcrypt_checkpw returned {}", ret);
|
||||
throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Internal failure decoding Bcrypt hash");
|
||||
return (ret == 0);
|
||||
#else
|
||||
throw Exception(
|
||||
@ -230,6 +233,17 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Password hash for the 'BCRYPT_PASSWORD' authentication type has length {} "
|
||||
"but must be 59 or 60 bytes.", hash.size());
|
||||
|
||||
auto resized = hash;
|
||||
resized.resize(64);
|
||||
|
||||
#if USE_BCRYPT
|
||||
/// Verify that it is a valid hash
|
||||
int ret = bcrypt_checkpw("", reinterpret_cast<const char *>(resized.data()));
|
||||
if (ret == -1)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Could not decode the provided hash with 'bcrypt_hash'");
|
||||
#endif
|
||||
|
||||
password_hash = hash;
|
||||
password_hash.resize(64);
|
||||
return;
|
||||
|
||||
@ -4,12 +4,12 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, AccessFlags access_flags_)
|
||||
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccessWrapper> & access_, AccessFlags access_flags_)
|
||||
: CachedAccessChecking(access_, AccessRightsElement{access_flags_})
|
||||
{
|
||||
}
|
||||
|
||||
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, const AccessRightsElement & element_)
|
||||
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccessWrapper> & access_, const AccessRightsElement & element_)
|
||||
: access(access_), element(element_)
|
||||
{
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Access/Common/AccessRightsElement.h>
|
||||
#include <Access/ContextAccess.h>
|
||||
#include <memory>
|
||||
|
||||
|
||||
@ -13,14 +14,14 @@ class ContextAccess;
|
||||
class CachedAccessChecking
|
||||
{
|
||||
public:
|
||||
CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, AccessFlags access_flags_);
|
||||
CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, const AccessRightsElement & element_);
|
||||
CachedAccessChecking(const std::shared_ptr<const ContextAccessWrapper> & access_, AccessFlags access_flags_);
|
||||
CachedAccessChecking(const std::shared_ptr<const ContextAccessWrapper> & access_, const AccessRightsElement & element_);
|
||||
~CachedAccessChecking();
|
||||
|
||||
bool checkAccess(bool throw_if_denied = true);
|
||||
|
||||
private:
|
||||
const std::shared_ptr<const ContextAccess> access;
|
||||
const std::shared_ptr<const ContextAccessWrapper> access;
|
||||
const AccessRightsElement element;
|
||||
bool checked = false;
|
||||
bool result = false;
|
||||
|
||||
@ -20,6 +20,7 @@
|
||||
#include <boost/algorithm/string/join.hpp>
|
||||
#include <boost/range/algorithm/set_algorithm.hpp>
|
||||
#include <cassert>
|
||||
#include <unordered_set>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -271,7 +272,7 @@ namespace
|
||||
|
||||
std::shared_ptr<const ContextAccess> ContextAccess::fromContext(const ContextPtr & context)
|
||||
{
|
||||
return context->getAccess();
|
||||
return ContextAccessWrapper::fromContext(context)->getAccess();
|
||||
}
|
||||
|
||||
|
||||
@ -560,7 +561,7 @@ std::shared_ptr<const AccessRights> ContextAccess::getAccessRightsWithImplicit()
|
||||
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... args) const
|
||||
bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, AccessFlags flags, const Args &... args) const
|
||||
{
|
||||
if (user_was_dropped)
|
||||
{
|
||||
@ -573,8 +574,10 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
if (params.full_access)
|
||||
return true;
|
||||
|
||||
auto access_granted = []
|
||||
auto access_granted = [&]
|
||||
{
|
||||
if constexpr (throw_if_denied)
|
||||
context->addQueryPrivilegesInfo(AccessRightsElement{flags, args...}.toStringWithoutOptions(), true);
|
||||
return true;
|
||||
};
|
||||
|
||||
@ -583,7 +586,10 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
FmtArgs && ...fmt_args [[maybe_unused]])
|
||||
{
|
||||
if constexpr (throw_if_denied)
|
||||
{
|
||||
context->addQueryPrivilegesInfo(AccessRightsElement{flags, args...}.toStringWithoutOptions(), false);
|
||||
throw Exception(error_code, std::move(fmt_string), getUserName(), std::forward<FmtArgs>(fmt_args)...);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
@ -686,102 +692,102 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool ContextAccess::checkAccessImpl(const AccessFlags & flags) const
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessFlags & flags) const
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(flags);
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(context, flags);
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool ContextAccess::checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessFlags & flags, std::string_view database, const Args &... args) const
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(flags, database.empty() ? params.current_database : database, args...);
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(context, flags, database.empty() ? params.current_database : database, args...);
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool ContextAccess::checkAccessImplHelper(const AccessRightsElement & element) const
|
||||
bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, const AccessRightsElement & element) const
|
||||
{
|
||||
assert(!element.grant_option || grant_option);
|
||||
if (element.isGlobalWithParameter())
|
||||
{
|
||||
if (element.any_parameter)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags);
|
||||
else
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags, element.parameter);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.parameter);
|
||||
}
|
||||
else if (element.any_database)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags);
|
||||
else if (element.any_table)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags, element.database);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.database);
|
||||
else if (element.any_column)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags, element.database, element.table);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.database, element.table);
|
||||
else
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags, element.database, element.table, element.columns);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.database, element.table, element.columns);
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool ContextAccess::checkAccessImpl(const AccessRightsElement & element) const
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessRightsElement & element) const
|
||||
{
|
||||
if constexpr (grant_option)
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, true>(element);
|
||||
return checkAccessImplHelper<throw_if_denied, true>(context, element);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (element.grant_option)
|
||||
return checkAccessImplHelper<throw_if_denied, true>(element);
|
||||
return checkAccessImplHelper<throw_if_denied, true>(context, element);
|
||||
else
|
||||
return checkAccessImplHelper<throw_if_denied, false>(element);
|
||||
return checkAccessImplHelper<throw_if_denied, false>(context, element);
|
||||
}
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessRightsElements & elements) const
|
||||
{
|
||||
for (const auto & element : elements)
|
||||
if (!checkAccessImpl<throw_if_denied, grant_option>(element))
|
||||
if (!checkAccessImpl<throw_if_denied, grant_option>(context, element))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl<false, false>(flags); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false>(flags, database); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false>(flags, database, table); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessRightsElement & element) const { return checkAccessImpl<false, false>(element); }
|
||||
bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return checkAccessImpl<false, false>(elements); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags) const { return checkAccessImpl<false, false>(context, flags); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false>(context, flags, database); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false>(context, flags, database, table); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false>(context, flags, database, table, column); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessRightsElement & element) const { return checkAccessImpl<false, false>(context, element); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessRightsElements & elements) const { return checkAccessImpl<false, false>(context, elements); }
|
||||
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags) const { return checkAccessImpl<false, true>(flags); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true>(flags, database); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true>(flags, database, table); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessRightsElement & element) const { return checkAccessImpl<false, true>(element); }
|
||||
bool ContextAccess::hasGrantOption(const AccessRightsElements & elements) const { return checkAccessImpl<false, true>(elements); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags) const { return checkAccessImpl<false, true>(context, flags); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true>(context, flags, database); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true>(context, flags, database, table); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true>(context, flags, database, table, column); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessRightsElement & element) const { return checkAccessImpl<false, true>(context, element); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const { return checkAccessImpl<false, true>(context, elements); }
|
||||
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl<true, false>(flags); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false>(flags, database); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false>(flags, database, table); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false>(flags, database, table, column); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl<true, false>(element); }
|
||||
void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl<true, false>(elements); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags) const { checkAccessImpl<true, false>(context, flags); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false>(context, flags, database); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false>(context, flags, database, table); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false>(context, flags, database, table, column); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessRightsElement & element) const { checkAccessImpl<true, false>(context, element); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessRightsElements & elements) const { checkAccessImpl<true, false>(context, elements); }
|
||||
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl<true, true>(flags); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true>(flags, database); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true>(flags, database, table); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true>(flags, database, table, column); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl<true, true>(element); }
|
||||
void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl<true, true>(elements); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags) const { checkAccessImpl<true, true>(context, flags); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true>(context, flags, database); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true>(context, flags, database, table); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true>(context, flags, database, table, column); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessRightsElement & element) const { checkAccessImpl<true, true>(context, element); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const { checkAccessImpl<true, true>(context, elements); }
|
||||
|
||||
|
||||
template <bool throw_if_denied, typename Container, typename GetNameFunction>
|
||||
bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const GetNameFunction & get_name_function) const
|
||||
bool ContextAccess::checkAdminOptionImplHelper(const ContextPtr & context, const Container & role_ids, const GetNameFunction & get_name_function) const
|
||||
{
|
||||
auto show_error = []<typename... FmtArgs>(int error_code [[maybe_unused]],
|
||||
FormatStringHelper<FmtArgs...> fmt_string [[maybe_unused]],
|
||||
@ -804,7 +810,7 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const
|
||||
if (!std::size(role_ids))
|
||||
return true;
|
||||
|
||||
if (isGranted(AccessType::ROLE_ADMIN))
|
||||
if (isGranted(context, AccessType::ROLE_ADMIN))
|
||||
return true;
|
||||
|
||||
auto info = getRolesInfo();
|
||||
@ -840,54 +846,54 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const UUID & role_id) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(to_array(role_id), [this](const UUID & id, size_t) { return access_control->tryReadName(id); });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, to_array(role_id), [this](const UUID & id, size_t) { return access_control->tryReadName(id); });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const UUID & role_id, const String & role_name) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id, const String & role_name) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(to_array(role_id), [&role_name](const UUID &, size_t) { return std::optional<String>{role_name}; });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, to_array(role_id), [&role_name](const UUID &, size_t) { return std::optional<String>{role_name}; });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(to_array(role_id), [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, to_array(role_id), [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const std::vector<UUID> & role_ids) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(role_ids, [this](const UUID & id, size_t) { return access_control->tryReadName(id); });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, role_ids, [this](const UUID & id, size_t) { return access_control->tryReadName(id); });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(role_ids, [&names_of_roles](const UUID &, size_t i) { return std::optional<String>{names_of_roles[i]}; });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, role_ids, [&names_of_roles](const UUID &, size_t i) { return std::optional<String>{names_of_roles[i]}; });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(role_ids, [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, role_ids, [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
}
|
||||
|
||||
bool ContextAccess::hasAdminOption(const UUID & role_id) const { return checkAdminOptionImpl<false>(role_id); }
|
||||
bool ContextAccess::hasAdminOption(const UUID & role_id, const String & role_name) const { return checkAdminOptionImpl<false>(role_id, role_name); }
|
||||
bool ContextAccess::hasAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { return checkAdminOptionImpl<false>(role_id, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const std::vector<UUID> & role_ids) const { return checkAdminOptionImpl<false>(role_ids); }
|
||||
bool ContextAccess::hasAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { return checkAdminOptionImpl<false>(role_ids, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { return checkAdminOptionImpl<false>(role_ids, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const UUID & role_id) const { return checkAdminOptionImpl<false>(context, role_id); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const UUID & role_id, const String & role_name) const { return checkAdminOptionImpl<false>(context, role_id, role_name); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { return checkAdminOptionImpl<false>(context, role_id, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids) const { return checkAdminOptionImpl<false>(context, role_ids); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { return checkAdminOptionImpl<false>(context, role_ids, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { return checkAdminOptionImpl<false>(context, role_ids, names_of_roles); }
|
||||
|
||||
void ContextAccess::checkAdminOption(const UUID & role_id) const { checkAdminOptionImpl<true>(role_id); }
|
||||
void ContextAccess::checkAdminOption(const UUID & role_id, const String & role_name) const { checkAdminOptionImpl<true>(role_id, role_name); }
|
||||
void ContextAccess::checkAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { checkAdminOptionImpl<true>(role_id, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids) const { checkAdminOptionImpl<true>(role_ids); }
|
||||
void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { checkAdminOptionImpl<true>(role_ids, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { checkAdminOptionImpl<true>(role_ids, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const UUID & role_id) const { checkAdminOptionImpl<true>(context, role_id); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const UUID & role_id, const String & role_name) const { checkAdminOptionImpl<true>(context, role_id, role_name); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { checkAdminOptionImpl<true>(context, role_id, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids) const { checkAdminOptionImpl<true>(context, role_ids); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { checkAdminOptionImpl<true>(context, role_ids, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { checkAdminOptionImpl<true>(context, role_ids, names_of_roles); }
|
||||
|
||||
|
||||
void ContextAccess::checkGranteeIsAllowed(const UUID & grantee_id, const IAccessEntity & grantee) const
|
||||
@ -919,4 +925,10 @@ void ContextAccess::checkGranteesAreAllowed(const std::vector<UUID> & grantee_id
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<const ContextAccessWrapper> ContextAccessWrapper::fromContext(const ContextPtr & context)
|
||||
{
|
||||
return context->getAccess();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -4,9 +4,12 @@
|
||||
#include <Access/ContextAccessParams.h>
|
||||
#include <Access/EnabledRowPolicies.h>
|
||||
#include <Interpreters/ClientInfo.h>
|
||||
#include <Access/QuotaUsage.h>
|
||||
#include <Common/SettingsChanges.h>
|
||||
#include <Core/UUID.h>
|
||||
#include <base/scope_guard.h>
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
@ -71,59 +74,59 @@ public:
|
||||
|
||||
/// Checks if a specified access is granted, and throws an exception if not.
|
||||
/// Empty database means the current database.
|
||||
void checkAccess(const AccessFlags & flags) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkAccess(const AccessRightsElement & element) const;
|
||||
void checkAccess(const AccessRightsElements & elements) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
void checkGrantOption(const AccessFlags & flags) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkGrantOption(const AccessRightsElement & element) const;
|
||||
void checkGrantOption(const AccessRightsElements & elements) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
/// Checks if a specified access is granted, and returns false if not.
|
||||
/// Empty database means the current database.
|
||||
bool isGranted(const AccessFlags & flags) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool isGranted(const AccessRightsElement & element) const;
|
||||
bool isGranted(const AccessRightsElements & elements) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
bool hasGrantOption(const AccessFlags & flags) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool hasGrantOption(const AccessRightsElement & element) const;
|
||||
bool hasGrantOption(const AccessRightsElements & elements) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
/// Checks if a specified role is granted with admin option, and throws an exception if not.
|
||||
void checkAdminOption(const UUID & role_id) const;
|
||||
void checkAdminOption(const UUID & role_id, const String & role_name) const;
|
||||
void checkAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
void checkAdminOption(const std::vector<UUID> & role_ids) const;
|
||||
void checkAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
void checkAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
void checkAdminOption(const ContextPtr & context, const UUID & role_id) const;
|
||||
void checkAdminOption(const ContextPtr & context, const UUID & role_id, const String & role_name) const;
|
||||
void checkAdminOption(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
void checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids) const;
|
||||
void checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
void checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
|
||||
/// Checks if a specified role is granted with admin option, and returns false if not.
|
||||
bool hasAdminOption(const UUID & role_id) const;
|
||||
bool hasAdminOption(const UUID & role_id, const String & role_name) const;
|
||||
bool hasAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool hasAdminOption(const std::vector<UUID> & role_ids) const;
|
||||
bool hasAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
bool hasAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const UUID & role_id) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const UUID & role_id, const String & role_name) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
|
||||
/// Checks if a grantee is allowed for the current user, throws an exception if not.
|
||||
void checkGranteeIsAllowed(const UUID & grantee_id, const IAccessEntity & grantee) const;
|
||||
@ -142,43 +145,43 @@ private:
|
||||
void calculateAccessRights() const TSA_REQUIRES(mutex);
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImpl(const AccessFlags & flags) const;
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const;
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessFlags & flags, std::string_view database, const Args &... args) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImpl(const AccessRightsElement & element) const;
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImpl(const AccessRightsElements & elements) const;
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool checkAccessImplHelper(AccessFlags flags, const Args &... args) const;
|
||||
bool checkAccessImplHelper(const ContextPtr & context, AccessFlags flags, const Args &... args) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImplHelper(const AccessRightsElement & element) const;
|
||||
bool checkAccessImplHelper(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const UUID & role_id) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const UUID & role_id, const String & role_name) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id, const String & role_name) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const std::vector<UUID> & role_ids) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
|
||||
template <bool throw_if_denied, typename Container, typename GetNameFunction>
|
||||
bool checkAdminOptionImplHelper(const Container & role_ids, const GetNameFunction & get_name_function) const;
|
||||
bool checkAdminOptionImplHelper(const ContextPtr & context, const Container & role_ids, const GetNameFunction & get_name_function) const;
|
||||
|
||||
const AccessControl * access_control = nullptr;
|
||||
const Params params;
|
||||
@ -203,4 +206,115 @@ private:
|
||||
mutable std::shared_ptr<const EnabledSettings> enabled_settings TSA_GUARDED_BY(mutex);
|
||||
};
|
||||
|
||||
/// This wrapper was added to be able to pass the current context to the access
|
||||
/// without the need to change the signature and all calls to the ContextAccess itself.
|
||||
/// Right now a context is used to store privileges that are checked for a query,
|
||||
/// and might be useful for something else in the future as well.
|
||||
class ContextAccessWrapper : public std::enable_shared_from_this<ContextAccessWrapper>
|
||||
{
|
||||
public:
|
||||
using ContextAccessPtr = std::shared_ptr<const ContextAccess>;
|
||||
|
||||
ContextAccessWrapper(const ContextAccessPtr & access_, const ContextPtr & context_): access(access_), context(context_) {}
|
||||
~ContextAccessWrapper() = default;
|
||||
|
||||
static std::shared_ptr<const ContextAccessWrapper> fromContext(const ContextPtr & context);
|
||||
|
||||
const ContextAccess::Params & getParams() const { return access->getParams(); }
|
||||
|
||||
const ContextAccessPtr & getAccess() const { return access; }
|
||||
|
||||
/// Returns the current user. Throws if user is nullptr.
|
||||
ALWAYS_INLINE UserPtr getUser() const { return access->getUser(); }
|
||||
/// Same as above, but can return nullptr.
|
||||
ALWAYS_INLINE UserPtr tryGetUser() const { return access->tryGetUser(); }
|
||||
ALWAYS_INLINE String getUserName() const { return access->getUserName(); }
|
||||
ALWAYS_INLINE std::optional<UUID> getUserID() const { return access->getUserID(); }
|
||||
|
||||
/// Returns information about current and enabled roles.
|
||||
ALWAYS_INLINE std::shared_ptr<const EnabledRolesInfo> getRolesInfo() const { return access->getRolesInfo(); }
|
||||
|
||||
/// Returns the row policy filter for a specified table.
|
||||
/// The function returns nullptr if there is no filter to apply.
|
||||
ALWAYS_INLINE RowPolicyFilterPtr getRowPolicyFilter(const String & database, const String & table_name, RowPolicyFilterType filter_type) const { return access->getRowPolicyFilter(database, table_name, filter_type); }
|
||||
|
||||
/// Returns the quota to track resource consumption.
|
||||
ALWAYS_INLINE std::shared_ptr<const EnabledQuota> getQuota() const { return access->getQuota(); }
|
||||
ALWAYS_INLINE std::optional<QuotaUsage> getQuotaUsage() const { return access->getQuotaUsage(); }
|
||||
|
||||
/// Returns the default settings, i.e. the settings which should be applied on user's login.
|
||||
ALWAYS_INLINE SettingsChanges getDefaultSettings() const { return access->getDefaultSettings(); }
|
||||
ALWAYS_INLINE std::shared_ptr<const SettingsProfilesInfo> getDefaultProfileInfo() const { return access->getDefaultProfileInfo(); }
|
||||
|
||||
/// Returns the current access rights.
|
||||
ALWAYS_INLINE std::shared_ptr<const AccessRights> getAccessRights() const { return access->getAccessRights(); }
|
||||
ALWAYS_INLINE std::shared_ptr<const AccessRights> getAccessRightsWithImplicit() const { return access->getAccessRightsWithImplicit(); }
|
||||
|
||||
/// Checks if a specified access is granted, and throws an exception if not.
|
||||
/// Empty database means the current database.
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags) const { access->checkAccess(context, flags); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database) const { access->checkAccess(context, flags, database); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { access->checkAccess(context, flags, database, table); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { access->checkAccess(context, flags, database, table, column); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { access->checkAccess(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { access->checkAccess(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessRightsElement & element) const { access->checkAccess(context, element); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessRightsElements & elements) const { access->checkAccess(context, elements); }
|
||||
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags) const { access->checkGrantOption(context, flags); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database) const { access->checkGrantOption(context, flags, database); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { access->checkGrantOption(context, flags, database, table); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { access->checkGrantOption(context, flags, database, table, column); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { access->checkGrantOption(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { access->checkGrantOption(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessRightsElement & element) const { access->checkGrantOption(context, element); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessRightsElements & elements) const { access->checkGrantOption(context, elements); }
|
||||
|
||||
/// Checks if a specified access is granted, and returns false if not.
|
||||
/// Empty database means the current database.
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags) const { return access->isGranted(context, flags); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database) const { return access->isGranted(context, flags, database); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return access->isGranted(context, flags, database, table); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return access->isGranted(context, flags, database, table, column); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return access->isGranted(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return access->isGranted(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessRightsElement & element) const { return access->isGranted(context, element); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessRightsElements & elements) const { return access->isGranted(context, elements); }
|
||||
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags) const { return access->hasGrantOption(context, flags); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database) const { return access->hasGrantOption(context, flags, database); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return access->hasGrantOption(context, flags, database, table); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return access->hasGrantOption(context, flags, database, table, column); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return access->hasGrantOption(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return access->hasGrantOption(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessRightsElement & element) const { return access->hasGrantOption(context, element); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessRightsElements & elements) const { return access->hasGrantOption(context, elements); }
|
||||
|
||||
/// Checks if a specified role is granted with admin option, and throws an exception if not.
|
||||
ALWAYS_INLINE void checkAdminOption(const UUID & role_id) const { access->checkAdminOption(context, role_id); }
|
||||
ALWAYS_INLINE void checkAdminOption(const UUID & role_id, const String & role_name) const { access->checkAdminOption(context, role_id, role_name); }
|
||||
ALWAYS_INLINE void checkAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { access->checkAdminOption(context, role_id, names_of_roles); }
|
||||
ALWAYS_INLINE void checkAdminOption(const std::vector<UUID> & role_ids) const { access->checkAdminOption(context, role_ids); }
|
||||
ALWAYS_INLINE void checkAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { access->checkAdminOption(context, role_ids, names_of_roles); }
|
||||
ALWAYS_INLINE void checkAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { access->checkAdminOption(context, role_ids, names_of_roles); }
|
||||
|
||||
/// Checks if a specified role is granted with admin option, and returns false if not.
|
||||
ALWAYS_INLINE bool hasAdminOption(const UUID & role_id) const { return access->hasAdminOption(context, role_id); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const UUID & role_id, const String & role_name) const { return access->hasAdminOption(context, role_id, role_name); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { return access->hasAdminOption(context, role_id, names_of_roles); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const std::vector<UUID> & role_ids) const { return access->hasAdminOption(context, role_ids); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { return access->hasAdminOption(context, role_ids, names_of_roles); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { return access->hasAdminOption(context, role_ids, names_of_roles); }
|
||||
|
||||
/// Checks if a grantee is allowed for the current user, throws an exception if not.
|
||||
ALWAYS_INLINE void checkGranteeIsAllowed(const UUID & grantee_id, const IAccessEntity & grantee) const { access->checkGranteeIsAllowed(grantee_id, grantee); }
|
||||
/// Checks if grantees are allowed for the current user, throws an exception if not.
|
||||
ALWAYS_INLINE void checkGranteesAreAllowed(const std::vector<UUID> & grantee_ids) const { access->checkGranteesAreAllowed(grantee_ids); }
|
||||
|
||||
private:
|
||||
ContextAccessPtr access;
|
||||
ContextPtr context;
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -194,11 +194,9 @@ DiskAccessStorage::DiskAccessStorage(const String & storage_name_, const String
|
||||
|
||||
DiskAccessStorage::~DiskAccessStorage()
|
||||
{
|
||||
stopListsWritingThread();
|
||||
|
||||
try
|
||||
{
|
||||
writeLists();
|
||||
DiskAccessStorage::shutdown();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -207,6 +205,17 @@ DiskAccessStorage::~DiskAccessStorage()
|
||||
}
|
||||
|
||||
|
||||
void DiskAccessStorage::shutdown()
|
||||
{
|
||||
stopListsWritingThread();
|
||||
|
||||
{
|
||||
std::lock_guard lock{mutex};
|
||||
writeLists();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
String DiskAccessStorage::getStorageParamsJSON() const
|
||||
{
|
||||
std::lock_guard lock{mutex};
|
||||
|
||||
@ -18,6 +18,8 @@ public:
|
||||
DiskAccessStorage(const String & storage_name_, const String & directory_path_, AccessChangesNotifier & changes_notifier_, bool readonly_, bool allow_backup_);
|
||||
~DiskAccessStorage() override;
|
||||
|
||||
void shutdown() override;
|
||||
|
||||
const char * getStorageType() const override { return STORAGE_TYPE; }
|
||||
String getStorageParamsJSON() const override;
|
||||
|
||||
|
||||
@ -44,6 +44,11 @@ public:
|
||||
explicit IAccessStorage(const String & storage_name_) : storage_name(storage_name_) {}
|
||||
virtual ~IAccessStorage() = default;
|
||||
|
||||
/// If the AccessStorage has to do some complicated work when destroying - do it in advance.
|
||||
/// For example, if the AccessStorage contains any threads for background work - ask them to complete and wait for completion.
|
||||
/// By default, does nothing.
|
||||
virtual void shutdown() {}
|
||||
|
||||
/// Returns the name of this storage.
|
||||
const String & getStorageName() const { return storage_name; }
|
||||
virtual const char * getStorageType() const = 0;
|
||||
|
||||
@ -34,11 +34,23 @@ MultipleAccessStorage::MultipleAccessStorage(const String & storage_name_)
|
||||
|
||||
MultipleAccessStorage::~MultipleAccessStorage()
|
||||
{
|
||||
/// It's better to remove the storages in the reverse order because they could depend on each other somehow.
|
||||
try
|
||||
{
|
||||
MultipleAccessStorage::shutdown();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
void MultipleAccessStorage::shutdown()
|
||||
{
|
||||
/// It's better to shutdown the storages in the reverse order because they could depend on each other somehow.
|
||||
const auto storages = getStoragesPtr();
|
||||
for (const auto & storage : *storages | boost::adaptors::reversed)
|
||||
{
|
||||
removeStorage(storage);
|
||||
storage->shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@ -72,6 +84,16 @@ void MultipleAccessStorage::removeStorage(const StoragePtr & storage_to_remove)
|
||||
ids_cache.clear();
|
||||
}
|
||||
|
||||
void MultipleAccessStorage::removeAllStorages()
|
||||
{
|
||||
/// It's better to remove the storages in the reverse order because they could depend on each other somehow.
|
||||
const auto storages = getStoragesPtr();
|
||||
for (const auto & storage : *storages | boost::adaptors::reversed)
|
||||
{
|
||||
removeStorage(storage);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<StoragePtr> MultipleAccessStorage::getStorages()
|
||||
{
|
||||
return *getStoragesPtr();
|
||||
|
||||
@ -21,6 +21,8 @@ public:
|
||||
explicit MultipleAccessStorage(const String & storage_name_ = STORAGE_TYPE);
|
||||
~MultipleAccessStorage() override;
|
||||
|
||||
void shutdown() override;
|
||||
|
||||
const char * getStorageType() const override { return STORAGE_TYPE; }
|
||||
bool isReadOnly() const override;
|
||||
bool isReadOnly(const UUID & id) const override;
|
||||
@ -32,6 +34,7 @@ public:
|
||||
void setStorages(const std::vector<StoragePtr> & storages);
|
||||
void addStorage(const StoragePtr & new_storage);
|
||||
void removeStorage(const StoragePtr & storage_to_remove);
|
||||
void removeAllStorages();
|
||||
std::vector<StoragePtr> getStorages();
|
||||
std::vector<ConstStoragePtr> getStorages() const;
|
||||
std::shared_ptr<const std::vector<StoragePtr>> getStoragesPtr();
|
||||
|
||||
@ -66,6 +66,18 @@ ReplicatedAccessStorage::ReplicatedAccessStorage(
|
||||
}
|
||||
|
||||
ReplicatedAccessStorage::~ReplicatedAccessStorage()
|
||||
{
|
||||
try
|
||||
{
|
||||
ReplicatedAccessStorage::shutdown();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
void ReplicatedAccessStorage::shutdown()
|
||||
{
|
||||
stopWatchingThread();
|
||||
}
|
||||
|
||||
@ -23,6 +23,8 @@ public:
|
||||
ReplicatedAccessStorage(const String & storage_name, const String & zookeeper_path, zkutil::GetZooKeeper get_zookeeper, AccessChangesNotifier & changes_notifier_, bool allow_backup);
|
||||
~ReplicatedAccessStorage() override;
|
||||
|
||||
void shutdown() override;
|
||||
|
||||
const char * getStorageType() const override { return STORAGE_TYPE; }
|
||||
|
||||
void startPeriodicReloading() override { startWatchingThread(); }
|
||||
|
||||
@ -15,22 +15,8 @@ namespace ErrorCodes
|
||||
|
||||
bool operator==(const SettingsProfilesInfo & lhs, const SettingsProfilesInfo & rhs)
|
||||
{
|
||||
if (lhs.settings != rhs.settings)
|
||||
return false;
|
||||
|
||||
if (lhs.constraints != rhs.constraints)
|
||||
return false;
|
||||
|
||||
if (lhs.profiles != rhs.profiles)
|
||||
return false;
|
||||
|
||||
if (lhs.profiles_with_implicit != rhs.profiles_with_implicit)
|
||||
return false;
|
||||
|
||||
if (lhs.names_of_profiles != rhs.names_of_profiles)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
return std::tie(lhs.settings, lhs.constraints, lhs.profiles, lhs.profiles_with_implicit, lhs.names_of_profiles)
|
||||
== std::tie(rhs.settings, rhs.constraints, rhs.profiles, rhs.profiles_with_implicit, rhs.names_of_profiles);
|
||||
}
|
||||
|
||||
std::shared_ptr<const SettingsConstraintsAndProfileIDs>
|
||||
@ -66,18 +52,20 @@ Strings SettingsProfilesInfo::getProfileNames() const
|
||||
{
|
||||
Strings result;
|
||||
result.reserve(profiles.size());
|
||||
for (const auto & profile_id : profiles)
|
||||
for (const UUID & profile_uuid : profiles)
|
||||
{
|
||||
const auto p = names_of_profiles.find(profile_id);
|
||||
if (p != names_of_profiles.end())
|
||||
result.push_back(p->second);
|
||||
const auto names_it = names_of_profiles.find(profile_uuid);
|
||||
if (names_it != names_of_profiles.end())
|
||||
{
|
||||
result.push_back(names_it->second);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (const auto name = access_control.tryReadName(profile_id))
|
||||
if (const auto name = access_control.tryReadName(profile_uuid))
|
||||
// We could've updated cache here, but it is a very rare case, so don't bother.
|
||||
result.push_back(*name);
|
||||
else
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unable to get profile name for {}", toString(profile_id));
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unable to get profile name for {}", toString(profile_uuid));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -29,7 +29,11 @@ struct SettingsProfilesInfo
|
||||
/// Names of all the profiles in `profiles`.
|
||||
std::unordered_map<UUID, String> names_of_profiles;
|
||||
|
||||
explicit SettingsProfilesInfo(const AccessControl & access_control_) : constraints(access_control_), access_control(access_control_) {}
|
||||
explicit SettingsProfilesInfo(const AccessControl & access_control_)
|
||||
: constraints(access_control_), access_control(access_control_)
|
||||
{
|
||||
}
|
||||
|
||||
std::shared_ptr<const SettingsConstraintsAndProfileIDs> getConstraintsAndProfileIDs(
|
||||
const std::shared_ptr<const SettingsConstraintsAndProfileIDs> & previous = nullptr) const;
|
||||
|
||||
|
||||
@ -228,6 +228,11 @@ public:
|
||||
return prefix_size + nested_func->sizeOfData();
|
||||
}
|
||||
|
||||
size_t alignOfData() const override
|
||||
{
|
||||
return std::max(alignof(Data), nested_func->alignOfData());
|
||||
}
|
||||
|
||||
void create(AggregateDataPtr __restrict place) const override
|
||||
{
|
||||
new (place) Data;
|
||||
|
||||
@ -985,18 +985,18 @@ std::string QueryAnalyzer::rewriteAggregateFunctionNameIfNeeded(
|
||||
{
|
||||
result_aggregate_function_name = settings.count_distinct_implementation;
|
||||
}
|
||||
else if (aggregate_function_name_lowercase == "countdistinctif" || aggregate_function_name_lowercase == "countifdistinct")
|
||||
else if (aggregate_function_name_lowercase == "countifdistinct" ||
|
||||
(settings.rewrite_count_distinct_if_with_count_distinct_implementation && aggregate_function_name_lowercase == "countdistinctif"))
|
||||
{
|
||||
result_aggregate_function_name = settings.count_distinct_implementation;
|
||||
result_aggregate_function_name += "If";
|
||||
}
|
||||
|
||||
/// Replace aggregateFunctionIfDistinct into aggregateFunctionDistinctIf to make execution more optimal
|
||||
if (result_aggregate_function_name.ends_with("ifdistinct"))
|
||||
else if (aggregate_function_name_lowercase.ends_with("ifdistinct"))
|
||||
{
|
||||
/// Replace aggregateFunctionIfDistinct into aggregateFunctionDistinctIf to make execution more optimal
|
||||
size_t prefix_length = result_aggregate_function_name.size() - strlen("ifdistinct");
|
||||
result_aggregate_function_name = result_aggregate_function_name.substr(0, prefix_length) + "DistinctIf";
|
||||
}
|
||||
}
|
||||
|
||||
bool need_add_or_null = settings.aggregate_functions_null_for_empty && !result_aggregate_function_name.ends_with("OrNull");
|
||||
if (need_add_or_null)
|
||||
|
||||
@ -9,6 +9,8 @@
|
||||
#include <Interpreters/convertFieldToType.h>
|
||||
#include <Interpreters/Set.h>
|
||||
|
||||
#include <Common/assert_cast.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -54,8 +56,9 @@ size_t getCompoundTypeDepth(const IDataType & type)
|
||||
}
|
||||
|
||||
template <typename Collection>
|
||||
Block createBlockFromCollection(const Collection & collection, const DataTypes & block_types, bool transform_null_in)
|
||||
Block createBlockFromCollection(const Collection & collection, const DataTypes& value_types, const DataTypes & block_types, bool transform_null_in)
|
||||
{
|
||||
assert(collection.size() == value_types.size());
|
||||
size_t columns_size = block_types.size();
|
||||
MutableColumns columns(columns_size);
|
||||
for (size_t i = 0; i < columns_size; ++i)
|
||||
@ -66,13 +69,17 @@ Block createBlockFromCollection(const Collection & collection, const DataTypes &
|
||||
|
||||
Row tuple_values;
|
||||
|
||||
for (const auto & value : collection)
|
||||
for (size_t collection_index = 0; collection_index < collection.size(); ++collection_index)
|
||||
{
|
||||
const auto & value = collection[collection_index];
|
||||
if (columns_size == 1)
|
||||
{
|
||||
auto field = convertFieldToTypeStrict(value, *block_types[0]);
|
||||
const DataTypePtr & data_type = value_types[collection_index];
|
||||
auto field = convertFieldToTypeStrict(value, *data_type, *block_types[0]);
|
||||
if (!field)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
bool need_insert_null = transform_null_in && block_types[0]->isNullable();
|
||||
if (!field->isNull() || need_insert_null)
|
||||
@ -87,6 +94,9 @@ Block createBlockFromCollection(const Collection & collection, const DataTypes &
|
||||
value.getTypeName());
|
||||
|
||||
const auto & tuple = value.template get<const Tuple &>();
|
||||
const DataTypePtr & value_type = value_types[collection_index];
|
||||
const DataTypes & tuple_value_type = typeid_cast<const DataTypeTuple *>(value_type.get())->getElements();
|
||||
|
||||
size_t tuple_size = tuple.size();
|
||||
|
||||
if (tuple_size != columns_size)
|
||||
@ -101,7 +111,7 @@ Block createBlockFromCollection(const Collection & collection, const DataTypes &
|
||||
size_t i = 0;
|
||||
for (; i < tuple_size; ++i)
|
||||
{
|
||||
auto converted_field = convertFieldToTypeStrict(tuple[i], *block_types[i]);
|
||||
auto converted_field = convertFieldToTypeStrict(tuple[i], *tuple_value_type[i], *block_types[i]);
|
||||
if (!converted_field)
|
||||
break;
|
||||
tuple_values[i] = std::move(*converted_field);
|
||||
@ -147,20 +157,28 @@ Block getSetElementsForConstantValue(const DataTypePtr & expression_type, const
|
||||
if (lhs_type_depth == rhs_type_depth)
|
||||
{
|
||||
/// 1 in 1; (1, 2) in (1, 2); identity(tuple(tuple(tuple(1)))) in tuple(tuple(tuple(1))); etc.
|
||||
|
||||
Array array{value};
|
||||
result_block = createBlockFromCollection(array, set_element_types, transform_null_in);
|
||||
DataTypes value_types{value_type};
|
||||
result_block = createBlockFromCollection(array, value_types, set_element_types, transform_null_in);
|
||||
}
|
||||
else if (lhs_type_depth + 1 == rhs_type_depth)
|
||||
{
|
||||
/// 1 in (1, 2); (1, 2) in ((1, 2), (3, 4))
|
||||
|
||||
WhichDataType rhs_which_type(value_type);
|
||||
|
||||
if (rhs_which_type.isArray())
|
||||
result_block = createBlockFromCollection(value.get<const Array &>(), set_element_types, transform_null_in);
|
||||
{
|
||||
const DataTypeArray * value_array_type = assert_cast<const DataTypeArray *>(value_type.get());
|
||||
size_t value_array_size = value.get<const Array &>().size();
|
||||
DataTypes value_types(value_array_size, value_array_type->getNestedType());
|
||||
result_block = createBlockFromCollection(value.get<const Array &>(), value_types, set_element_types, transform_null_in);
|
||||
}
|
||||
else if (rhs_which_type.isTuple())
|
||||
result_block = createBlockFromCollection(value.get<const Tuple &>(), set_element_types, transform_null_in);
|
||||
{
|
||||
const DataTypeTuple * value_tuple_type = assert_cast<const DataTypeTuple *>(value_type.get());
|
||||
const DataTypes & value_types = value_tuple_type->getElements();
|
||||
result_block = createBlockFromCollection(value.get<const Tuple &>(), value_types, set_element_types, transform_null_in);
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Unsupported type at the right-side of IN. Expected Array or Tuple. Actual {}",
|
||||
|
||||
@ -24,8 +24,6 @@
|
||||
#include <Poco/Util/XMLConfiguration.h>
|
||||
#include <Poco/DOM/DOMParser.h>
|
||||
|
||||
#include <ranges>
|
||||
|
||||
|
||||
namespace ProfileEvents
|
||||
{
|
||||
@ -93,6 +91,7 @@ BackupImpl::BackupImpl(
|
||||
const std::optional<BackupInfo> & base_backup_info_,
|
||||
std::shared_ptr<IBackupReader> reader_,
|
||||
const ContextPtr & context_,
|
||||
bool is_internal_backup_,
|
||||
bool use_same_s3_credentials_for_base_backup_)
|
||||
: backup_info(backup_info_)
|
||||
, backup_name_for_logging(backup_info.toStringForLogging())
|
||||
@ -101,7 +100,7 @@ BackupImpl::BackupImpl(
|
||||
, open_mode(OpenMode::READ)
|
||||
, reader(std::move(reader_))
|
||||
, context(context_)
|
||||
, is_internal_backup(false)
|
||||
, is_internal_backup(is_internal_backup_)
|
||||
, version(INITIAL_BACKUP_VERSION)
|
||||
, base_backup_info(base_backup_info_)
|
||||
, use_same_s3_credentials_for_base_backup(use_same_s3_credentials_for_base_backup_)
|
||||
@ -256,6 +255,7 @@ std::shared_ptr<const IBackup> BackupImpl::getBaseBackupUnlocked() const
|
||||
params.backup_info = *base_backup_info;
|
||||
params.open_mode = OpenMode::READ;
|
||||
params.context = context;
|
||||
params.is_internal_backup = is_internal_backup;
|
||||
/// use_same_s3_credentials_for_base_backup should be inherited for base backups
|
||||
params.use_same_s3_credentials_for_base_backup = use_same_s3_credentials_for_base_backup;
|
||||
|
||||
|
||||
@ -40,6 +40,7 @@ public:
|
||||
const std::optional<BackupInfo> & base_backup_info_,
|
||||
std::shared_ptr<IBackupReader> reader_,
|
||||
const ContextPtr & context_,
|
||||
bool is_internal_backup_,
|
||||
bool use_same_s3_credentials_for_base_backup_);
|
||||
|
||||
BackupImpl(
|
||||
|
||||
@ -153,6 +153,7 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
|
||||
params.base_backup_info,
|
||||
reader,
|
||||
params.context,
|
||||
params.is_internal_backup,
|
||||
/* use_same_s3_credentials_for_base_backup*/ false);
|
||||
}
|
||||
else
|
||||
|
||||
@ -119,6 +119,7 @@ void registerBackupEngineS3(BackupFactory & factory)
|
||||
params.base_backup_info,
|
||||
reader,
|
||||
params.context,
|
||||
params.is_internal_backup,
|
||||
params.use_same_s3_credentials_for_base_backup);
|
||||
}
|
||||
else
|
||||
|
||||
@ -177,6 +177,7 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
|
||||
params.base_backup_info,
|
||||
reader,
|
||||
params.context,
|
||||
params.is_internal_backup,
|
||||
params.use_same_s3_credentials_for_base_backup);
|
||||
}
|
||||
else
|
||||
|
||||
@ -44,13 +44,12 @@
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTColumnDeclaration.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/Kusto/ParserKQLStatement.h>
|
||||
#include <Parsers/PRQL/ParserPRQLQuery.h>
|
||||
#include <Parsers/Kusto/ParserKQLStatement.h>
|
||||
#include <Parsers/Kusto/parseKQLQuery.h>
|
||||
|
||||
#include <Processors/Formats/Impl/NullFormat.h>
|
||||
#include <Processors/Formats/IInputFormat.h>
|
||||
#include <Processors/Formats/IOutputFormat.h>
|
||||
#include <Processors/QueryPlan/QueryPlan.h>
|
||||
#include <Processors/QueryPlan/BuildQueryPipelineSettings.h>
|
||||
#include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
|
||||
|
||||
@ -255,6 +255,17 @@ void HedgedConnections::sendCancel()
|
||||
if (!sent_query || cancelled)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot cancel. Either no query sent or already cancelled.");
|
||||
|
||||
/// All hedged connections should be stopped, since otherwise before the
|
||||
/// HedgedConnectionsFactory will be destroyed (that will happen from
|
||||
/// QueryPipeline dtor) they could still do some work.
|
||||
/// And not only this does not make sense, but it also could lead to
|
||||
/// use-after-free of the current_thread, since the thread from which they
|
||||
/// had been created differs from the thread where the dtor of
|
||||
/// QueryPipeline will be called and the initial thread could be already
|
||||
/// destroyed (especially when the system is under pressure).
|
||||
if (hedged_connections_factory.hasEventsInProcess())
|
||||
hedged_connections_factory.stopChoosingReplicas();
|
||||
|
||||
cancelled = true;
|
||||
|
||||
for (auto & offset_status : offset_states)
|
||||
|
||||
@ -415,6 +415,15 @@ Value saveAllArenasMetric(AsynchronousMetricValues & values,
|
||||
fmt::format("jemalloc.arenas.all.{}", metric_name));
|
||||
}
|
||||
|
||||
template<typename Value>
|
||||
Value saveJemallocProf(AsynchronousMetricValues & values,
|
||||
const std::string & metric_name)
|
||||
{
|
||||
return saveJemallocMetricImpl<Value>(values,
|
||||
fmt::format("prof.{}", metric_name),
|
||||
fmt::format("jemalloc.prof.{}", metric_name));
|
||||
}
|
||||
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -607,6 +616,7 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update)
|
||||
saveJemallocMetric<size_t>(new_values, "background_thread.num_threads");
|
||||
saveJemallocMetric<uint64_t>(new_values, "background_thread.num_runs");
|
||||
saveJemallocMetric<uint64_t>(new_values, "background_thread.run_intervals");
|
||||
saveJemallocProf<size_t>(new_values, "active");
|
||||
saveAllArenasMetric<size_t>(new_values, "pactive");
|
||||
[[maybe_unused]] size_t je_malloc_pdirty = saveAllArenasMetric<size_t>(new_values, "pdirty");
|
||||
[[maybe_unused]] size_t je_malloc_pmuzzy = saveAllArenasMetric<size_t>(new_values, "pmuzzy");
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#include <Common/CurrentMetrics.h>
|
||||
|
||||
|
||||
// clang-format off
|
||||
/// Available metrics. Add something here as you wish.
|
||||
/// If the metric is generic (i.e. not server specific)
|
||||
/// it should be also added to src/Coordination/KeeperConstant.cpp
|
||||
|
||||
@ -202,7 +202,10 @@ uint64_t readU64(std::string_view & sp)
|
||||
{
|
||||
SAFE_CHECK(sp.size() >= N, "underflow");
|
||||
uint64_t x = 0;
|
||||
memcpy(&x, sp.data(), N);
|
||||
if constexpr (std::endian::native == std::endian::little)
|
||||
memcpy(&x, sp.data(), N);
|
||||
else
|
||||
memcpy(reinterpret_cast<char*>(&x) + sizeof(uint64_t) - N, sp.data(), N);
|
||||
sp.remove_prefix(N);
|
||||
return x;
|
||||
}
|
||||
|
||||
@ -602,6 +602,8 @@
|
||||
M(721, DEPRECATED_FUNCTION) \
|
||||
M(722, ASYNC_LOAD_WAIT_FAILED) \
|
||||
M(723, PARQUET_EXCEPTION) \
|
||||
M(724, TOO_MANY_TABLES) \
|
||||
M(725, TOO_MANY_DATABASES) \
|
||||
\
|
||||
M(900, DISTRIBUTED_CACHE_ERROR) \
|
||||
M(901, CANNOT_USE_DISTRIBUTED_CACHE) \
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
#include <mutex>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
@ -35,7 +33,7 @@ namespace ErrorCodes
|
||||
|
||||
struct Error
|
||||
{
|
||||
/// Number of times Exception with this ErrorCode had been throw.
|
||||
/// Number of times Exception with this ErrorCode has been thrown.
|
||||
Value count = 0;
|
||||
/// Time of the last error.
|
||||
UInt64 error_time_ms = 0;
|
||||
|
||||
161
src/Common/HilbertUtils.h
Normal file
161
src/Common/HilbertUtils.h
Normal file
@ -0,0 +1,161 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <Common/BitHelpers.h>
|
||||
#include "base/types.h"
|
||||
#include <Functions/hilbertDecode2DLUT.h>
|
||||
#include <base/defines.h>
|
||||
#include <array>
|
||||
#include <set>
|
||||
|
||||
|
||||
namespace HilbertDetails
|
||||
{
|
||||
|
||||
struct Segment // represents [begin; end], all bounds are included
|
||||
{
|
||||
UInt64 begin;
|
||||
UInt64 end;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
Given the range of values of hilbert code - and this function will return segments of the Hilbert curve
|
||||
such that each of them lies in a whole domain (aka square)
|
||||
0 1
|
||||
┌────────────────────────────────┐
|
||||
│ │ │
|
||||
│ │ │
|
||||
0 │ 00xxx │ 11xxx │
|
||||
│ | │ | │
|
||||
│ | │ | │
|
||||
│_______________│________________│
|
||||
│ | │ | │
|
||||
│ | │ | │
|
||||
│ | │ | │
|
||||
1 │ 01xxx______│_____10xxx │
|
||||
│ │ │
|
||||
│ │ │
|
||||
└────────────────────────────────┘
|
||||
Imagine a square, one side of which is a x-axis, other is a y-axis.
|
||||
First approximation of the Hilbert curve is on the picture - U curve.
|
||||
So we divide Hilbert Code Interval on 4 parts each of which is represented by a square
|
||||
and look where the given interval [start, finish] is located:
|
||||
[00xxxxxx | 01xxxxxx | 10xxxxxx | 11xxxxxx ]
|
||||
1: [ ]
|
||||
start = 0010111 end = 10111110
|
||||
2: [ ] [ ]
|
||||
If it contains a whole sector (that represents a domain=square),
|
||||
then we take this range. In the example above - it is a sector [01000000, 01111111]
|
||||
Then we dig into the recursion and check the remaining ranges.
|
||||
Note that after the first call all other ranges in the recursion will have either start or finish on the end of a range,
|
||||
so the complexity of the algorithm will be O(logN), where N is the maximum of hilbert code.
|
||||
*/
|
||||
template <typename F>
|
||||
void segmentBinaryPartition(UInt64 start, UInt64 finish, UInt8 current_bits, F && callback)
|
||||
{
|
||||
if (current_bits == 0)
|
||||
return;
|
||||
|
||||
const auto next_bits = current_bits - 2;
|
||||
const auto history = current_bits == 64 ? 0 : (start >> current_bits) << current_bits;
|
||||
|
||||
const auto chunk_mask = 0b11;
|
||||
const auto start_chunk = (start >> next_bits) & chunk_mask;
|
||||
const auto finish_chunk = (finish >> next_bits) & chunk_mask;
|
||||
|
||||
auto construct_range = [next_bits, history](UInt64 chunk)
|
||||
{
|
||||
return HilbertDetails::Segment{
|
||||
.begin = history + (chunk << next_bits),
|
||||
.end = history + ((chunk + 1) << next_bits) - 1
|
||||
};
|
||||
};
|
||||
|
||||
if (start_chunk == finish_chunk)
|
||||
{
|
||||
if ((finish - start + 1) == (1 << next_bits)) // it means that [begin, end] is a range
|
||||
{
|
||||
callback(HilbertDetails::Segment{.begin = start, .end = finish});
|
||||
return;
|
||||
}
|
||||
segmentBinaryPartition(start, finish, next_bits, callback);
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto range_chunk = start_chunk + 1; range_chunk < finish_chunk; ++range_chunk)
|
||||
{
|
||||
callback(construct_range(range_chunk));
|
||||
}
|
||||
|
||||
const auto start_range = construct_range(start_chunk);
|
||||
if (start == start_range.begin)
|
||||
{
|
||||
callback(start_range);
|
||||
}
|
||||
else
|
||||
{
|
||||
segmentBinaryPartition(start, start_range.end, next_bits, callback);
|
||||
}
|
||||
|
||||
const auto finish_range = construct_range(finish_chunk);
|
||||
if (finish == finish_range.end)
|
||||
{
|
||||
callback(finish_range);
|
||||
}
|
||||
else
|
||||
{
|
||||
segmentBinaryPartition(finish_range.begin, finish, next_bits, callback);
|
||||
}
|
||||
}
|
||||
|
||||
// Given 2 points representing ends of the range of Hilbert Curve that lies in a whole domain.
|
||||
// The are neighbour corners of some square - and the function returns ranges of both sides of this square
|
||||
inline std::array<std::pair<UInt64, UInt64>, 2> createRangeFromCorners(UInt64 x1, UInt64 y1, UInt64 x2, UInt64 y2)
|
||||
{
|
||||
UInt64 dist_x = x1 > x2 ? x1 - x2 : x2 - x1;
|
||||
UInt64 dist_y = y1 > y2 ? y1 - y2 : y2 - y1;
|
||||
UInt64 range_size = std::max(dist_x, dist_y);
|
||||
bool contains_minimum_vertice = x1 % (range_size + 1) == 0;
|
||||
if (contains_minimum_vertice)
|
||||
{
|
||||
UInt64 x_min = std::min(x1, x2);
|
||||
UInt64 y_min = std::min(y1, y2);
|
||||
return {
|
||||
std::pair<UInt64, UInt64>{x_min, x_min + range_size},
|
||||
std::pair<UInt64, UInt64>{y_min, y_min + range_size}
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
UInt64 x_max = std::max(x1, x2);
|
||||
UInt64 y_max = std::max(y1, y2);
|
||||
chassert(x_max >= range_size);
|
||||
chassert(y_max >= range_size);
|
||||
return {
|
||||
std::pair<UInt64, UInt64>{x_max - range_size, x_max},
|
||||
std::pair<UInt64, UInt64>{y_max - range_size, y_max}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Unpack an interval of Hilbert curve to hyperrectangles covered by it across N dimensions.
|
||||
*/
|
||||
template <typename F>
|
||||
void hilbertIntervalToHyperrectangles2D(UInt64 first, UInt64 last, F && callback)
|
||||
{
|
||||
const auto equal_bits_count = getLeadingZeroBits(last | first);
|
||||
const auto even_equal_bits_count = equal_bits_count - equal_bits_count % 2;
|
||||
segmentBinaryPartition(first, last, 64 - even_equal_bits_count, [&](HilbertDetails::Segment range)
|
||||
{
|
||||
auto interval1 = DB::FunctionHilbertDecode2DWIthLookupTableImpl<3>::decode(range.begin);
|
||||
auto interval2 = DB::FunctionHilbertDecode2DWIthLookupTableImpl<3>::decode(range.end);
|
||||
|
||||
std::array<std::pair<UInt64, UInt64>, 2> unpacked_range = createRangeFromCorners(
|
||||
std::get<0>(interval1), std::get<1>(interval1),
|
||||
std::get<0>(interval2), std::get<1>(interval2));
|
||||
|
||||
callback(unpacked_range);
|
||||
});
|
||||
}
|
||||
@ -48,7 +48,7 @@ public:
|
||||
|
||||
/// HashFunction usually hashes the entire key and the found key will be equal the provided key. In such cases, use get(). It is also
|
||||
/// possible to store other, non-hashed data in the key. In that case, the found key is potentially different from the provided key.
|
||||
/// Then use getWithKey() to also return the found key including it's non-hashed data.
|
||||
/// Then use getWithKey() to also return the found key including its non-hashed data.
|
||||
virtual MappedPtr get(const Key & key) = 0;
|
||||
virtual std::optional<KeyMapped> getWithKey(const Key &) = 0;
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <Common/TraceSender.h>
|
||||
|
||||
|
||||
// clang-format off
|
||||
/// Available events. Add something here as you wish.
|
||||
/// If the event is generic (i.e. not server specific)
|
||||
/// it should be also added to src/Coordination/KeeperConstant.cpp
|
||||
@ -14,6 +15,7 @@
|
||||
M(QueriesWithSubqueries, "Count queries with all subqueries") \
|
||||
M(SelectQueriesWithSubqueries, "Count SELECT queries with all subqueries") \
|
||||
M(InsertQueriesWithSubqueries, "Count INSERT queries with all subqueries") \
|
||||
M(SelectQueriesWithPrimaryKeyUsage, "Count SELECT queries which use the primary key to evaluate the WHERE condition") \
|
||||
M(AsyncInsertQuery, "Same as InsertQuery, but only for asynchronous INSERT queries.") \
|
||||
M(AsyncInsertBytes, "Data size in bytes of asynchronous INSERT queries.") \
|
||||
M(AsyncInsertRows, "Number of rows inserted by asynchronous INSERT queries.") \
|
||||
|
||||
@ -11,10 +11,10 @@
|
||||
#include <Poco/Util/XMLConfiguration.h>
|
||||
|
||||
#include <boost/noncopyable.hpp>
|
||||
#include <boost/intrusive/list.hpp>
|
||||
|
||||
#include <chrono>
|
||||
#include <deque>
|
||||
#include <queue>
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
@ -30,6 +30,8 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
class ISchedulerNode;
|
||||
class EventQueue;
|
||||
using EventId = UInt64;
|
||||
|
||||
inline const Poco::Util::AbstractConfiguration & emptyConfig()
|
||||
{
|
||||
@ -82,6 +84,115 @@ struct SchedulerNodeInfo
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/*
|
||||
* Node of hierarchy for scheduling requests for resource. Base class for all
|
||||
* kinds of scheduling elements (queues, policies, constraints and schedulers).
|
||||
*
|
||||
* Root node is a scheduler, which has it's thread to dequeue requests,
|
||||
* execute requests (see ResourceRequest) and process events in a thread-safe manner.
|
||||
* Immediate children of the scheduler represent independent resources.
|
||||
* Each resource has it's own hierarchy to achieve required scheduling policies.
|
||||
* Non-leaf nodes do not hold requests, but keep scheduling state
|
||||
* (e.g. consumption history, amount of in-flight requests, etc).
|
||||
* Leafs of hierarchy are queues capable of holding pending requests.
|
||||
*
|
||||
* scheduler (SchedulerRoot)
|
||||
* / \
|
||||
* constraint constraint (SemaphoreConstraint)
|
||||
* | |
|
||||
* policy policy (PriorityPolicy)
|
||||
* / \ / \
|
||||
* q1 q2 q3 q4 (FifoQueue)
|
||||
*
|
||||
* Dequeueing request from an inner node will dequeue request from one of active leaf-queues in its subtree.
|
||||
* Node is considered to be active iff:
|
||||
* - it has at least one pending request in one of leaves of it's subtree;
|
||||
* - and enforced constraints, if any, are satisfied
|
||||
* (e.g. amount of concurrent requests is not greater than some number).
|
||||
*
|
||||
* All methods must be called only from scheduler thread for thread-safety.
|
||||
*/
|
||||
class ISchedulerNode : public boost::intrusive::list_base_hook<>, private boost::noncopyable
|
||||
{
|
||||
public:
|
||||
explicit ISchedulerNode(EventQueue * event_queue_, const Poco::Util::AbstractConfiguration & config = emptyConfig(), const String & config_prefix = {})
|
||||
: event_queue(event_queue_)
|
||||
, info(config, config_prefix)
|
||||
{}
|
||||
|
||||
virtual ~ISchedulerNode() = default;
|
||||
|
||||
/// Checks if two nodes configuration is equal
|
||||
virtual bool equals(ISchedulerNode * other)
|
||||
{
|
||||
return info.equals(other->info);
|
||||
}
|
||||
|
||||
/// Attach new child
|
||||
virtual void attachChild(const std::shared_ptr<ISchedulerNode> & child) = 0;
|
||||
|
||||
/// Detach and destroy child
|
||||
virtual void removeChild(ISchedulerNode * child) = 0;
|
||||
|
||||
/// Get attached child by name
|
||||
virtual ISchedulerNode * getChild(const String & child_name) = 0;
|
||||
|
||||
/// Activation of child due to the first pending request
|
||||
/// Should be called on leaf node (i.e. queue) to propagate activation signal through chain to the root
|
||||
virtual void activateChild(ISchedulerNode * child) = 0;
|
||||
|
||||
/// Returns true iff node is active
|
||||
virtual bool isActive() = 0;
|
||||
|
||||
/// Returns number of active children
|
||||
virtual size_t activeChildren() = 0;
|
||||
|
||||
/// Returns the first request to be executed as the first component of resulting pair.
|
||||
/// The second pair component is `true` iff node is still active after dequeueing.
|
||||
virtual std::pair<ResourceRequest *, bool> dequeueRequest() = 0;
|
||||
|
||||
/// Returns full path string using names of every parent
|
||||
String getPath()
|
||||
{
|
||||
String result;
|
||||
ISchedulerNode * ptr = this;
|
||||
while (ptr->parent)
|
||||
{
|
||||
result = "/" + ptr->basename + result;
|
||||
ptr = ptr->parent;
|
||||
}
|
||||
return result.empty() ? "/" : result;
|
||||
}
|
||||
|
||||
/// Attach to a parent (used by attachChild)
|
||||
virtual void setParent(ISchedulerNode * parent_)
|
||||
{
|
||||
parent = parent_;
|
||||
}
|
||||
|
||||
protected:
|
||||
/// Notify parents about the first pending request or constraint becoming satisfied.
|
||||
/// Postponed to be handled in scheduler thread, so it is intended to be called from outside.
|
||||
void scheduleActivation();
|
||||
|
||||
public:
|
||||
EventQueue * const event_queue;
|
||||
String basename;
|
||||
SchedulerNodeInfo info;
|
||||
ISchedulerNode * parent = nullptr;
|
||||
EventId activation_event_id = 0; // Valid for `ISchedulerNode` placed in EventQueue::activations
|
||||
|
||||
/// Introspection
|
||||
std::atomic<UInt64> dequeued_requests{0};
|
||||
std::atomic<UInt64> canceled_requests{0};
|
||||
std::atomic<ResourceCost> dequeued_cost{0};
|
||||
std::atomic<ResourceCost> canceled_cost{0};
|
||||
std::atomic<UInt64> busy_periods{0};
|
||||
};
|
||||
|
||||
using SchedulerNodePtr = std::shared_ptr<ISchedulerNode>;
|
||||
|
||||
/*
|
||||
* Simple waitable thread-safe FIFO task queue.
|
||||
* Intended to hold postponed events for later handling (usually by scheduler thread).
|
||||
@ -89,57 +200,70 @@ struct SchedulerNodeInfo
|
||||
class EventQueue
|
||||
{
|
||||
public:
|
||||
using Event = std::function<void()>;
|
||||
using Task = std::function<void()>;
|
||||
|
||||
static constexpr EventId not_postponed = 0;
|
||||
|
||||
using TimePoint = std::chrono::system_clock::time_point;
|
||||
using Duration = std::chrono::system_clock::duration;
|
||||
static constexpr UInt64 not_postponed = 0;
|
||||
|
||||
struct Event
|
||||
{
|
||||
const EventId event_id;
|
||||
Task task;
|
||||
|
||||
Event(EventId event_id_, Task && task_)
|
||||
: event_id(event_id_)
|
||||
, task(std::move(task_))
|
||||
{}
|
||||
};
|
||||
|
||||
struct Postponed
|
||||
{
|
||||
TimePoint key;
|
||||
UInt64 id; // for canceling
|
||||
std::unique_ptr<Event> event;
|
||||
EventId event_id; // for canceling
|
||||
std::unique_ptr<Task> task;
|
||||
|
||||
Postponed(TimePoint key_, UInt64 id_, Event && event_)
|
||||
Postponed(TimePoint key_, EventId event_id_, Task && task_)
|
||||
: key(key_)
|
||||
, id(id_)
|
||||
, event(std::make_unique<Event>(std::move(event_)))
|
||||
, event_id(event_id_)
|
||||
, task(std::make_unique<Task>(std::move(task_)))
|
||||
{}
|
||||
|
||||
bool operator<(const Postponed & rhs) const
|
||||
{
|
||||
return std::tie(key, id) > std::tie(rhs.key, rhs.id); // reversed for min-heap
|
||||
return std::tie(key, event_id) > std::tie(rhs.key, rhs.event_id); // reversed for min-heap
|
||||
}
|
||||
};
|
||||
|
||||
/// Add an `event` to be processed after `until` time point.
|
||||
/// Returns a unique id for canceling.
|
||||
[[nodiscard]] UInt64 postpone(TimePoint until, Event && event)
|
||||
/// Returns a unique event id for canceling.
|
||||
[[nodiscard]] EventId postpone(TimePoint until, Task && task)
|
||||
{
|
||||
std::unique_lock lock{mutex};
|
||||
if (postponed.empty() || until < postponed.front().key)
|
||||
pending.notify_one();
|
||||
auto id = ++last_id;
|
||||
postponed.emplace_back(until, id, std::move(event));
|
||||
auto event_id = ++last_event_id;
|
||||
postponed.emplace_back(until, event_id, std::move(task));
|
||||
std::push_heap(postponed.begin(), postponed.end());
|
||||
return id;
|
||||
return event_id;
|
||||
}
|
||||
|
||||
/// Cancel a postponed event using its unique id.
|
||||
/// NOTE: Only postponed events can be canceled.
|
||||
/// NOTE: If you need to cancel enqueued event, consider doing your actions inside another enqueued
|
||||
/// NOTE: event instead. This ensures that all previous events are processed.
|
||||
bool cancelPostponed(UInt64 postponed_id)
|
||||
bool cancelPostponed(EventId postponed_event_id)
|
||||
{
|
||||
if (postponed_id == not_postponed)
|
||||
if (postponed_event_id == not_postponed)
|
||||
return false;
|
||||
std::unique_lock lock{mutex};
|
||||
for (auto i = postponed.begin(), e = postponed.end(); i != e; ++i)
|
||||
{
|
||||
if (i->id == postponed_id)
|
||||
if (i->event_id == postponed_event_id)
|
||||
{
|
||||
postponed.erase(i);
|
||||
// It is O(n), but we do not expect either big heaps or frequent cancels. So it is fine.
|
||||
// It is O(n), but we do not expect neither big heaps nor frequent cancels. So it is fine.
|
||||
std::make_heap(postponed.begin(), postponed.end());
|
||||
return true;
|
||||
}
|
||||
@ -148,11 +272,23 @@ public:
|
||||
}
|
||||
|
||||
/// Add an `event` for immediate processing
|
||||
void enqueue(Event && event)
|
||||
void enqueue(Task && task)
|
||||
{
|
||||
std::unique_lock lock{mutex};
|
||||
bool was_empty = queue.empty();
|
||||
queue.emplace_back(event);
|
||||
bool was_empty = events.empty() && activations.empty();
|
||||
auto event_id = ++last_event_id;
|
||||
events.emplace_back(event_id, std::move(task));
|
||||
if (was_empty)
|
||||
pending.notify_one();
|
||||
}
|
||||
|
||||
/// Add an activation `event` for immediate processing. Activations use a separate queue for performance reasons.
|
||||
void enqueueActivation(ISchedulerNode * node)
|
||||
{
|
||||
std::unique_lock lock{mutex};
|
||||
bool was_empty = events.empty() && activations.empty();
|
||||
node->activation_event_id = ++last_event_id;
|
||||
activations.push_back(*node);
|
||||
if (was_empty)
|
||||
pending.notify_one();
|
||||
}
|
||||
@ -163,7 +299,7 @@ public:
|
||||
bool forceProcess()
|
||||
{
|
||||
std::unique_lock lock{mutex};
|
||||
if (!queue.empty())
|
||||
if (!events.empty() || !activations.empty())
|
||||
{
|
||||
processQueue(std::move(lock));
|
||||
return true;
|
||||
@ -181,7 +317,7 @@ public:
|
||||
bool tryProcess()
|
||||
{
|
||||
std::unique_lock lock{mutex};
|
||||
if (!queue.empty())
|
||||
if (!events.empty() || !activations.empty())
|
||||
{
|
||||
processQueue(std::move(lock));
|
||||
return true;
|
||||
@ -205,7 +341,7 @@ public:
|
||||
std::unique_lock lock{mutex};
|
||||
while (true)
|
||||
{
|
||||
if (!queue.empty())
|
||||
if (!events.empty() || !activations.empty())
|
||||
{
|
||||
processQueue(std::move(lock));
|
||||
return;
|
||||
@ -269,141 +405,69 @@ private:
|
||||
|
||||
void processQueue(std::unique_lock<std::mutex> && lock)
|
||||
{
|
||||
Event event = std::move(queue.front());
|
||||
queue.pop_front();
|
||||
if (events.empty())
|
||||
{
|
||||
processActivation(std::move(lock));
|
||||
return;
|
||||
}
|
||||
if (activations.empty())
|
||||
{
|
||||
processEvent(std::move(lock));
|
||||
return;
|
||||
}
|
||||
if (activations.front().activation_event_id < events.front().event_id)
|
||||
processActivation(std::move(lock));
|
||||
else
|
||||
processEvent(std::move(lock));
|
||||
}
|
||||
|
||||
void processActivation(std::unique_lock<std::mutex> && lock)
|
||||
{
|
||||
ISchedulerNode * node = &activations.front();
|
||||
activations.pop_front();
|
||||
node->activation_event_id = 0;
|
||||
lock.unlock(); // do not hold queue mutex while processing events
|
||||
event();
|
||||
node->parent->activateChild(node);
|
||||
}
|
||||
|
||||
void processEvent(std::unique_lock<std::mutex> && lock)
|
||||
{
|
||||
Task task = std::move(events.front().task);
|
||||
events.pop_front();
|
||||
lock.unlock(); // do not hold queue mutex while processing events
|
||||
task();
|
||||
}
|
||||
|
||||
void processPostponed(std::unique_lock<std::mutex> && lock)
|
||||
{
|
||||
Event event = std::move(*postponed.front().event);
|
||||
Task task = std::move(*postponed.front().task);
|
||||
std::pop_heap(postponed.begin(), postponed.end());
|
||||
postponed.pop_back();
|
||||
lock.unlock(); // do not hold queue mutex while processing events
|
||||
event();
|
||||
task();
|
||||
}
|
||||
|
||||
std::mutex mutex;
|
||||
std::condition_variable pending;
|
||||
std::deque<Event> queue;
|
||||
|
||||
// `events` and `activations` logically represent one ordered queue. To preserve the common order we use `EventId`
|
||||
// Activations are stored in a separate queue for performance reasons (mostly to avoid any allocations)
|
||||
std::deque<Event> events;
|
||||
boost::intrusive::list<ISchedulerNode> activations;
|
||||
|
||||
std::vector<Postponed> postponed;
|
||||
UInt64 last_id = 0;
|
||||
EventId last_event_id = 0;
|
||||
|
||||
std::atomic<TimePoint> manual_time{TimePoint()}; // for tests only
|
||||
};
|
||||
|
||||
/*
|
||||
* Node of hierarchy for scheduling requests for resource. Base class for all
|
||||
* kinds of scheduling elements (queues, policies, constraints and schedulers).
|
||||
*
|
||||
* Root node is a scheduler, which has it's thread to dequeue requests,
|
||||
* execute requests (see ResourceRequest) and process events in a thread-safe manner.
|
||||
* Immediate children of the scheduler represent independent resources.
|
||||
* Each resource has it's own hierarchy to achieve required scheduling policies.
|
||||
* Non-leaf nodes do not hold requests, but keep scheduling state
|
||||
* (e.g. consumption history, amount of in-flight requests, etc).
|
||||
* Leafs of hierarchy are queues capable of holding pending requests.
|
||||
*
|
||||
* scheduler (SchedulerRoot)
|
||||
* / \
|
||||
* constraint constraint (SemaphoreConstraint)
|
||||
* | |
|
||||
* policy policy (PriorityPolicy)
|
||||
* / \ / \
|
||||
* q1 q2 q3 q4 (FifoQueue)
|
||||
*
|
||||
* Dequeueing request from an inner node will dequeue request from one of active leaf-queues in its subtree.
|
||||
* Node is considered to be active iff:
|
||||
* - it has at least one pending request in one of leaves of it's subtree;
|
||||
* - and enforced constraints, if any, are satisfied
|
||||
* (e.g. amount of concurrent requests is not greater than some number).
|
||||
*
|
||||
* All methods must be called only from scheduler thread for thread-safety.
|
||||
*/
|
||||
class ISchedulerNode : private boost::noncopyable
|
||||
inline void ISchedulerNode::scheduleActivation()
|
||||
{
|
||||
public:
|
||||
explicit ISchedulerNode(EventQueue * event_queue_, const Poco::Util::AbstractConfiguration & config = emptyConfig(), const String & config_prefix = {})
|
||||
: event_queue(event_queue_)
|
||||
, info(config, config_prefix)
|
||||
{}
|
||||
|
||||
virtual ~ISchedulerNode() = default;
|
||||
|
||||
/// Checks if two nodes configuration is equal
|
||||
virtual bool equals(ISchedulerNode * other)
|
||||
if (likely(parent))
|
||||
{
|
||||
return info.equals(other->info);
|
||||
// The same as `enqueue([this] { parent->activateChild(this); });` but faster
|
||||
event_queue->enqueueActivation(this);
|
||||
}
|
||||
|
||||
/// Attach new child
|
||||
virtual void attachChild(const std::shared_ptr<ISchedulerNode> & child) = 0;
|
||||
|
||||
/// Detach and destroy child
|
||||
virtual void removeChild(ISchedulerNode * child) = 0;
|
||||
|
||||
/// Get attached child by name
|
||||
virtual ISchedulerNode * getChild(const String & child_name) = 0;
|
||||
|
||||
/// Activation of child due to the first pending request
|
||||
/// Should be called on leaf node (i.e. queue) to propagate activation signal through chain to the root
|
||||
virtual void activateChild(ISchedulerNode * child) = 0;
|
||||
|
||||
/// Returns true iff node is active
|
||||
virtual bool isActive() = 0;
|
||||
|
||||
/// Returns number of active children
|
||||
virtual size_t activeChildren() = 0;
|
||||
|
||||
/// Returns the first request to be executed as the first component of resulting pair.
|
||||
/// The second pair component is `true` iff node is still active after dequeueing.
|
||||
virtual std::pair<ResourceRequest *, bool> dequeueRequest() = 0;
|
||||
|
||||
/// Returns full path string using names of every parent
|
||||
String getPath()
|
||||
{
|
||||
String result;
|
||||
ISchedulerNode * ptr = this;
|
||||
while (ptr->parent)
|
||||
{
|
||||
result = "/" + ptr->basename + result;
|
||||
ptr = ptr->parent;
|
||||
}
|
||||
return result.empty() ? "/" : result;
|
||||
}
|
||||
|
||||
/// Attach to a parent (used by attachChild)
|
||||
virtual void setParent(ISchedulerNode * parent_)
|
||||
{
|
||||
parent = parent_;
|
||||
}
|
||||
|
||||
protected:
|
||||
/// Notify parents about the first pending request or constraint becoming satisfied.
|
||||
/// Postponed to be handled in scheduler thread, so it is intended to be called from outside.
|
||||
void scheduleActivation()
|
||||
{
|
||||
if (likely(parent))
|
||||
{
|
||||
event_queue->enqueue([this] { parent->activateChild(this); });
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
EventQueue * const event_queue;
|
||||
String basename;
|
||||
SchedulerNodeInfo info;
|
||||
ISchedulerNode * parent = nullptr;
|
||||
|
||||
/// Introspection
|
||||
std::atomic<UInt64> dequeued_requests{0};
|
||||
std::atomic<UInt64> canceled_requests{0};
|
||||
std::atomic<ResourceCost> dequeued_cost{0};
|
||||
std::atomic<ResourceCost> canceled_cost{0};
|
||||
std::atomic<UInt64> busy_periods{0};
|
||||
};
|
||||
|
||||
using SchedulerNodePtr = std::shared_ptr<ISchedulerNode>;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
143
src/Common/Scheduler/Nodes/tests/gtest_event_queue.cpp
Normal file
143
src/Common/Scheduler/Nodes/tests/gtest_event_queue.cpp
Normal file
@ -0,0 +1,143 @@
|
||||
#include <chrono>
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <Common/Scheduler/ISchedulerNode.h>
|
||||
|
||||
using namespace DB;
|
||||
|
||||
class FakeSchedulerNode : public ISchedulerNode
|
||||
{
|
||||
public:
|
||||
explicit FakeSchedulerNode(String & log_, EventQueue * event_queue_, const Poco::Util::AbstractConfiguration & config = emptyConfig(), const String & config_prefix = {})
|
||||
: ISchedulerNode(event_queue_, config, config_prefix)
|
||||
, log(log_)
|
||||
{}
|
||||
|
||||
void attachChild(const SchedulerNodePtr & child) override
|
||||
{
|
||||
log += " +" + child->basename;
|
||||
}
|
||||
|
||||
void removeChild(ISchedulerNode * child) override
|
||||
{
|
||||
log += " -" + child->basename;
|
||||
}
|
||||
|
||||
ISchedulerNode * getChild(const String & /* child_name */) override
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void activateChild(ISchedulerNode * child) override
|
||||
{
|
||||
log += " A" + child->basename;
|
||||
}
|
||||
|
||||
bool isActive() override
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
size_t activeChildren() override
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::pair<ResourceRequest *, bool> dequeueRequest() override
|
||||
{
|
||||
log += " D";
|
||||
return {nullptr, false};
|
||||
}
|
||||
|
||||
private:
|
||||
String & log;
|
||||
};
|
||||
|
||||
struct QueueTest {
|
||||
String log;
|
||||
EventQueue event_queue;
|
||||
FakeSchedulerNode root_node;
|
||||
|
||||
QueueTest()
|
||||
: root_node(log, &event_queue)
|
||||
{}
|
||||
|
||||
SchedulerNodePtr makeNode(const String & name)
|
||||
{
|
||||
auto node = std::make_shared<FakeSchedulerNode>(log, &event_queue);
|
||||
node->basename = name;
|
||||
node->setParent(&root_node);
|
||||
return std::static_pointer_cast<ISchedulerNode>(node);
|
||||
}
|
||||
|
||||
void process(EventQueue::TimePoint now, const String & expected_log, size_t limit = size_t(-1))
|
||||
{
|
||||
event_queue.setManualTime(now);
|
||||
for (;limit > 0; limit--)
|
||||
{
|
||||
if (!event_queue.tryProcess())
|
||||
break;
|
||||
}
|
||||
EXPECT_EQ(log, expected_log);
|
||||
log.clear();
|
||||
}
|
||||
|
||||
void activate(const SchedulerNodePtr & node)
|
||||
{
|
||||
event_queue.enqueueActivation(node.get());
|
||||
}
|
||||
|
||||
void event(const String & text)
|
||||
{
|
||||
event_queue.enqueue([this, text] { log += " " + text; });
|
||||
}
|
||||
|
||||
EventId postpone(EventQueue::TimePoint until, const String & text)
|
||||
{
|
||||
return event_queue.postpone(until, [this, text] { log += " " + text; });
|
||||
}
|
||||
|
||||
void cancel(EventId event_id)
|
||||
{
|
||||
event_queue.cancelPostponed(event_id);
|
||||
}
|
||||
};
|
||||
|
||||
TEST(SchedulerEventQueue, Smoke)
|
||||
{
|
||||
QueueTest t;
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
EventQueue::TimePoint start = std::chrono::system_clock::now();
|
||||
t.process(start, "", 0);
|
||||
|
||||
// Activations
|
||||
auto node1 = t.makeNode("1");
|
||||
auto node2 = t.makeNode("2");
|
||||
t.activate(node2);
|
||||
t.activate(node1);
|
||||
t.process(start + 42s, " A2 A1");
|
||||
|
||||
// Events
|
||||
t.event("E1");
|
||||
t.event("E2");
|
||||
t.process(start + 100s, " E1 E2");
|
||||
|
||||
// Postponed events
|
||||
t.postpone(start + 200s, "P200");
|
||||
auto p190 = t.postpone(start + 200s, "P190");
|
||||
t.postpone(start + 150s, "P150");
|
||||
t.postpone(start + 175s, "P175");
|
||||
t.process(start + 180s, " P150 P175");
|
||||
t.event("E3");
|
||||
t.cancel(p190);
|
||||
t.process(start + 300s, " E3 P200");
|
||||
|
||||
// Ordering of events and activations
|
||||
t.event("E1");
|
||||
t.activate(node1);
|
||||
t.event("E2");
|
||||
t.activate(node2);
|
||||
t.process(start + 300s, " E1 A1 E2 A2");
|
||||
}
|
||||
@ -5,8 +5,6 @@
|
||||
|
||||
#include <Common/Scheduler/Nodes/FairPolicy.h>
|
||||
#include <Common/Scheduler/Nodes/ThrottlerConstraint.h>
|
||||
#include "Common/Scheduler/ISchedulerNode.h"
|
||||
#include "Common/Scheduler/ResourceRequest.h"
|
||||
|
||||
using namespace DB;
|
||||
|
||||
|
||||
@ -85,9 +85,18 @@ StatusFile::StatusFile(std::string path_, FillFunction fill_)
|
||||
|
||||
/// Write information about current server instance to the file.
|
||||
WriteBufferFromFileDescriptor out(fd, 1024);
|
||||
fill(out);
|
||||
/// Finalize here to avoid throwing exceptions in destructor.
|
||||
out.finalize();
|
||||
try
|
||||
{
|
||||
fill(out);
|
||||
/// Finalize here to avoid throwing exceptions in destructor.
|
||||
out.finalize();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
/// Finalize in case of exception to avoid throwing exceptions in destructor
|
||||
out.finalize();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include <Interpreters/AsynchronousMetricLog.h>
|
||||
#include <Interpreters/CrashLog.h>
|
||||
#include <Interpreters/ErrorLog.h>
|
||||
#include <Interpreters/MetricLog.h>
|
||||
#include <Interpreters/OpenTelemetrySpanLog.h>
|
||||
#include <Interpreters/PartLog.h>
|
||||
|
||||
@ -1,9 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <memory>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
#include <base/types.h>
|
||||
|
||||
@ -32,7 +30,8 @@
|
||||
M(FilesystemReadPrefetchesLogElement) \
|
||||
M(AsynchronousInsertLogElement) \
|
||||
M(BackupLogElement) \
|
||||
M(BlobStorageLogElement)
|
||||
M(BlobStorageLogElement) \
|
||||
M(ErrorLogElement)
|
||||
|
||||
namespace Poco
|
||||
{
|
||||
|
||||
@ -609,7 +609,10 @@ void KeeperStorage::UncommittedState::commit(int64_t commit_zxid)
|
||||
uncommitted_auth.pop_front();
|
||||
if (uncommitted_auth.empty())
|
||||
session_and_auth.erase(add_auth->session_id);
|
||||
|
||||
}
|
||||
else if (auto * close_session = std::get_if<CloseSessionDelta>(&front_delta.operation))
|
||||
{
|
||||
closed_sessions.erase(close_session->session_id);
|
||||
}
|
||||
|
||||
deltas.pop_front();
|
||||
@ -682,6 +685,10 @@ void KeeperStorage::UncommittedState::rollback(int64_t rollback_zxid)
|
||||
session_and_auth.erase(add_auth->session_id);
|
||||
}
|
||||
}
|
||||
else if (auto * close_session = std::get_if<CloseSessionDelta>(&delta_it->operation))
|
||||
{
|
||||
closed_sessions.erase(close_session->session_id);
|
||||
}
|
||||
}
|
||||
|
||||
if (delta_it == deltas.rend())
|
||||
@ -878,6 +885,10 @@ Coordination::Error KeeperStorage::commit(int64_t commit_zxid)
|
||||
session_and_auth[operation.session_id].emplace_back(std::move(operation.auth_id));
|
||||
return Coordination::Error::ZOK;
|
||||
}
|
||||
else if constexpr (std::same_as<DeltaType, KeeperStorage::CloseSessionDelta>)
|
||||
{
|
||||
return Coordination::Error::ZOK;
|
||||
}
|
||||
else
|
||||
{
|
||||
// shouldn't be called in any process functions
|
||||
@ -2366,12 +2377,15 @@ void KeeperStorage::preprocessRequest(
|
||||
|
||||
ephemerals.erase(session_ephemerals);
|
||||
}
|
||||
new_deltas.emplace_back(transaction.zxid, CloseSessionDelta{session_id});
|
||||
uncommitted_state.closed_sessions.insert(session_id);
|
||||
|
||||
new_digest = calculateNodesDigest(new_digest, new_deltas);
|
||||
return;
|
||||
}
|
||||
|
||||
if (check_acl && !request_processor->checkAuth(*this, session_id, false))
|
||||
if ((check_acl && !request_processor->checkAuth(*this, session_id, false)) ||
|
||||
uncommitted_state.closed_sessions.contains(session_id)) // Is session closed but not committed yet
|
||||
{
|
||||
uncommitted_state.deltas.emplace_back(new_last_zxid, Coordination::Error::ZNOAUTH);
|
||||
return;
|
||||
|
||||
@ -314,8 +314,13 @@ public:
|
||||
AuthID auth_id;
|
||||
};
|
||||
|
||||
struct CloseSessionDelta
|
||||
{
|
||||
int64_t session_id;
|
||||
};
|
||||
|
||||
using Operation = std::
|
||||
variant<CreateNodeDelta, RemoveNodeDelta, UpdateNodeDelta, SetACLDelta, AddAuthDelta, ErrorDelta, SubDeltaEnd, FailedMultiDelta>;
|
||||
variant<CreateNodeDelta, RemoveNodeDelta, UpdateNodeDelta, SetACLDelta, AddAuthDelta, ErrorDelta, SubDeltaEnd, FailedMultiDelta, CloseSessionDelta>;
|
||||
|
||||
struct Delta
|
||||
{
|
||||
@ -351,6 +356,7 @@ public:
|
||||
std::shared_ptr<Node> tryGetNodeFromStorage(StringRef path) const;
|
||||
|
||||
std::unordered_map<int64_t, std::list<const AuthID *>> session_and_auth;
|
||||
std::unordered_set<int64_t> closed_sessions;
|
||||
|
||||
struct UncommittedNode
|
||||
{
|
||||
|
||||
@ -2019,6 +2019,67 @@ TEST_P(CoordinationTest, TestCreateNodeWithAuthSchemeForAclWhenAuthIsPrecommitte
|
||||
EXPECT_EQ(acls[0].permissions, 31);
|
||||
}
|
||||
|
||||
TEST_P(CoordinationTest, TestPreprocessWhenCloseSessionIsPrecommitted)
|
||||
{
|
||||
using namespace Coordination;
|
||||
using namespace DB;
|
||||
|
||||
ChangelogDirTest snapshots("./snapshots");
|
||||
setSnapshotDirectory("./snapshots");
|
||||
ResponsesQueue queue(std::numeric_limits<size_t>::max());
|
||||
SnapshotsQueue snapshots_queue{1};
|
||||
int64_t session_id = 1;
|
||||
size_t term = 0;
|
||||
|
||||
auto state_machine = std::make_shared<KeeperStateMachine>(queue, snapshots_queue, keeper_context, nullptr);
|
||||
state_machine->init();
|
||||
|
||||
auto & storage = state_machine->getStorageUnsafe();
|
||||
const auto & uncommitted_state = storage.uncommitted_state;
|
||||
|
||||
// Create first node for the session
|
||||
String node_path_1 = "/node_1";
|
||||
std::shared_ptr<ZooKeeperCreateRequest> create_req_1 = std::make_shared<ZooKeeperCreateRequest>();
|
||||
create_req_1->path = node_path_1;
|
||||
auto create_entry_1 = getLogEntryFromZKRequest(term, session_id, state_machine->getNextZxid(), create_req_1);
|
||||
|
||||
state_machine->pre_commit(1, create_entry_1->get_buf());
|
||||
EXPECT_TRUE(uncommitted_state.nodes.contains(node_path_1));
|
||||
|
||||
state_machine->commit(1, create_entry_1->get_buf());
|
||||
EXPECT_TRUE(storage.container.contains(node_path_1));
|
||||
|
||||
// Close session
|
||||
std::shared_ptr<ZooKeeperCloseRequest> close_req = std::make_shared<ZooKeeperCloseRequest>();
|
||||
auto close_entry = getLogEntryFromZKRequest(term, session_id, state_machine->getNextZxid(), close_req);
|
||||
// Pre-commit close session
|
||||
state_machine->pre_commit(2, close_entry->get_buf());
|
||||
|
||||
// Try to create second node after close session is pre-committed
|
||||
String node_path_2 = "/node_2";
|
||||
std::shared_ptr<ZooKeeperCreateRequest> create_req_2 = std::make_shared<ZooKeeperCreateRequest>();
|
||||
create_req_2->path = node_path_2;
|
||||
auto create_entry_2 = getLogEntryFromZKRequest(term, session_id, state_machine->getNextZxid(), create_req_2);
|
||||
|
||||
// Pre-commit creating second node
|
||||
state_machine->pre_commit(3, create_entry_2->get_buf());
|
||||
// Second node wasn't created
|
||||
EXPECT_FALSE(uncommitted_state.nodes.contains(node_path_2));
|
||||
|
||||
// Rollback pre-committed closing session
|
||||
state_machine->rollback(3, create_entry_2->get_buf());
|
||||
state_machine->rollback(2, close_entry->get_buf());
|
||||
|
||||
// Pre-commit creating second node
|
||||
state_machine->pre_commit(2, create_entry_2->get_buf());
|
||||
// Now second node was created
|
||||
EXPECT_TRUE(uncommitted_state.nodes.contains(node_path_2));
|
||||
|
||||
state_machine->commit(2, create_entry_2->get_buf());
|
||||
EXPECT_TRUE(storage.container.contains(node_path_1));
|
||||
EXPECT_TRUE(storage.container.contains(node_path_2));
|
||||
}
|
||||
|
||||
TEST_P(CoordinationTest, TestSetACLWithAuthSchemeForAclWhenAuthIsPrecommitted)
|
||||
{
|
||||
using namespace Coordination;
|
||||
|
||||
@ -14,6 +14,7 @@ class AbstractConfiguration;
|
||||
namespace DB
|
||||
{
|
||||
|
||||
// clang-format off
|
||||
#define SERVER_SETTINGS(M, ALIAS) \
|
||||
M(Bool, show_addresses_in_stack_traces, true, "If it is set true will show addresses in stack traces", 0) \
|
||||
M(Bool, shutdown_wait_unfinished_queries, false, "If set true ClickHouse will wait for running queries finish before shutdown.", 0) \
|
||||
@ -85,7 +86,7 @@ namespace DB
|
||||
M(Double, index_mark_cache_size_ratio, DEFAULT_INDEX_MARK_CACHE_SIZE_RATIO, "The size of the protected queue in the secondary index mark cache relative to the cache's total size.", 0) \
|
||||
M(UInt64, page_cache_chunk_size, 2 << 20, "Bytes per chunk in userspace page cache. Rounded up to a multiple of page size (typically 4 KiB) or huge page size (typically 2 MiB, only if page_cache_use_thp is enabled).", 0) \
|
||||
M(UInt64, page_cache_mmap_size, 1 << 30, "Bytes per memory mapping in userspace page cache. Not important.", 0) \
|
||||
M(UInt64, page_cache_size, 10ul << 30, "Amount of virtual memory to map for userspace page cache. If page_cache_use_madv_free is enabled, it's recommended to set this higher than the machine's RAM size. Use 0 to disable userspace page cache.", 0) \
|
||||
M(UInt64, page_cache_size, 0, "Amount of virtual memory to map for userspace page cache. If page_cache_use_madv_free is enabled, it's recommended to set this higher than the machine's RAM size. Use 0 to disable userspace page cache.", 0) \
|
||||
M(Bool, page_cache_use_madv_free, DBMS_DEFAULT_PAGE_CACHE_USE_MADV_FREE, "If true, the userspace page cache will allow the OS to automatically reclaim memory from the cache on memory pressure (using MADV_FREE).", 0) \
|
||||
M(Bool, page_cache_use_transparent_huge_pages, true, "Userspace will attempt to use transparent huge pages on Linux. This is best-effort.", 0) \
|
||||
M(UInt64, mmap_cache_size, DEFAULT_MMAP_CACHE_MAX_SIZE, "A cache for mmapped files.", 0) \
|
||||
@ -102,6 +103,8 @@ namespace DB
|
||||
M(UInt64, max_dictionary_num_to_warn, 1000lu, "If the number of dictionaries is greater than this value, the server will create a warning that will displayed to user.", 0) \
|
||||
M(UInt64, max_database_num_to_warn, 1000lu, "If the number of databases is greater than this value, the server will create a warning that will displayed to user.", 0) \
|
||||
M(UInt64, max_part_num_to_warn, 100000lu, "If the number of parts is greater than this value, the server will create a warning that will displayed to user.", 0) \
|
||||
M(UInt64, max_table_num_to_throw, 0lu, "If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \
|
||||
M(UInt64, max_database_num_to_throw, 0lu, "If number of databases is greater than this value, server will throw an exception. 0 means no limitation.", 0) \
|
||||
M(UInt64, concurrent_threads_soft_limit_num, 0, "Sets how many concurrent thread can be allocated before applying CPU pressure. Zero means unlimited.", 0) \
|
||||
M(UInt64, concurrent_threads_soft_limit_ratio_to_cores, 0, "Same as concurrent_threads_soft_limit_num, but with ratio to cores.", 0) \
|
||||
\
|
||||
@ -146,6 +149,8 @@ namespace DB
|
||||
M(UInt64, global_profiler_real_time_period_ns, 0, "Period for real clock timer of global profiler (in nanoseconds). Set 0 value to turn off the real clock global profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
|
||||
M(UInt64, global_profiler_cpu_time_period_ns, 0, "Period for CPU clock timer of global profiler (in nanoseconds). Set 0 value to turn off the CPU clock global profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
|
||||
M(Bool, enable_azure_sdk_logging, false, "Enables logging from Azure sdk", 0) \
|
||||
M(String, merge_workload, "default", "Name of workload to be used to access resources for all merges (may be overridden by a merge tree setting)", 0) \
|
||||
M(String, mutation_workload, "default", "Name of workload to be used to access resources for all mutations (may be overridden by a merge tree setting)", 0) \
|
||||
M(Double, gwp_asan_force_sample_probability, 0, "Probability that an allocation from specific places will be sampled by GWP Asan (i.e. PODArray allocations)", 0) \
|
||||
M(Bool, use_legacy_mongodb_integration, false, "Use the legacy MongoDB integration implementation. Deprecated.", 0) \
|
||||
|
||||
|
||||
@ -31,6 +31,7 @@ class IColumn;
|
||||
* for tracking settings changes in different versions and for special `compatibility` setting to work correctly.
|
||||
*/
|
||||
|
||||
// clang-format off
|
||||
#define COMMON_SETTINGS(M, ALIAS) \
|
||||
M(Dialect, dialect, Dialect::clickhouse, "Which dialect will be used to parse query", 0)\
|
||||
M(UInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.", 0) \
|
||||
@ -933,6 +934,7 @@ class IColumn;
|
||||
M(Int64, prefer_warmed_unmerged_parts_seconds, 0, "Only available in ClickHouse Cloud. If a merged part is less than this many seconds old and is not pre-warmed (see cache_populated_by_fetch), but all its source parts are available and pre-warmed, SELECT queries will read from those parts instead. Only for ReplicatedMergeTree. Note that this only checks whether CacheWarmer processed the part; if the part was fetched into cache by something else, it'll still be considered cold until CacheWarmer gets to it; if it was warmed, then evicted from cache, it'll still be considered warm.", 0) \
|
||||
M(Bool, iceberg_engine_ignore_schema_evolution, false, "Ignore schema evolution in Iceberg table engine and read all data using latest schema saved on table creation. Note that it can lead to incorrect result", 0) \
|
||||
M(Bool, allow_deprecated_error_prone_window_functions, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)", 0) \
|
||||
M(Bool, uniform_snowflake_conversion_functions, true, "Enables functions snowflakeIDToDateTime[64] and dateTime[64]ToSnowflakeID while disabling functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake.", 0) \
|
||||
M(Bool, mongodb_fail_on_query_build_error, true, "If enabled, MongoDB tables will return an error when a MongoDB query can't be built. Not applied for the legacy implementation, or if the new analyzer is used.", 0) \
|
||||
|
||||
// End of COMMON_SETTINGS
|
||||
@ -1145,7 +1147,9 @@ class IColumn;
|
||||
M(UInt64, output_format_pretty_max_value_width, 10000, "Maximum width of value to display in Pretty formats. If greater - it will be cut.", 0) \
|
||||
M(UInt64, output_format_pretty_max_value_width_apply_for_single_value, false, "Only cut values (see the `output_format_pretty_max_value_width` setting) when it is not a single value in a block. Otherwise output it entirely, which is useful for the `SHOW CREATE TABLE` query.", 0) \
|
||||
M(UInt64Auto, output_format_pretty_color, "auto", "Use ANSI escape sequences in Pretty formats. 0 - disabled, 1 - enabled, 'auto' - enabled if a terminal.", 0) \
|
||||
M(String, output_format_pretty_grid_charset, "UTF-8", "Charset for printing grid borders. Available charsets: ASCII, UTF-8 (default one).", 0) \
|
||||
M(String, output_format_pretty_grid_charset, "UTF-8", "Charset for printing grid borders. Available charsets: ASCII, UTF-8 (default one).", 0) \
|
||||
M(UInt64, output_format_pretty_display_footer_column_names, true, "Display column names in the footer if there are 999 or more rows.", 0) \
|
||||
M(UInt64, output_format_pretty_display_footer_column_names_min_rows, 50, "Sets the minimum threshold value of rows for which to enable displaying column names in the footer. 50 (default)", 0) \
|
||||
M(UInt64, output_format_parquet_row_group_size, 1000000, "Target row group size in rows.", 0) \
|
||||
M(UInt64, output_format_parquet_row_group_size_bytes, 512 * 1024 * 1024, "Target row group size in bytes, before compression.", 0) \
|
||||
M(Bool, output_format_parquet_string_as_string, true, "Use Parquet String type instead of Binary for String columns.", 0) \
|
||||
|
||||
@ -75,6 +75,7 @@ namespace SettingsChangesHistory
|
||||
using SettingsChanges = std::vector<SettingChange>;
|
||||
}
|
||||
|
||||
// clang-format off
|
||||
/// History of settings changes that controls some backward incompatible changes
|
||||
/// across all ClickHouse versions. It maps ClickHouse version to settings changes that were done
|
||||
/// in this version. This history contains both changes to existing settings and newly added settings.
|
||||
@ -101,6 +102,7 @@ static const std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges
|
||||
{"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."},
|
||||
{"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."},
|
||||
{"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"},
|
||||
{"uniform_snowflake_conversion_functions", false, true, "Enable functions snowflakeIDToDateTime[64] and dateTime[64]ToSnowflakeID."},
|
||||
{"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."},
|
||||
{"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."},
|
||||
{"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."},
|
||||
@ -108,6 +110,8 @@ static const std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges
|
||||
{"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"},
|
||||
{"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"},
|
||||
{"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"},
|
||||
{"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."},
|
||||
{"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."},
|
||||
{"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
|
||||
{"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
|
||||
{"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
|
||||
|
||||
@ -146,7 +146,7 @@ void SerializationVariantElement::deserializeBinaryBulkWithMultipleStreams(
|
||||
}
|
||||
|
||||
/// If we started to read a new column, reinitialize variant column in deserialization state.
|
||||
if (!variant_element_state->variant || result_column->empty())
|
||||
if (!variant_element_state->variant || mutable_column->empty())
|
||||
{
|
||||
variant_element_state->variant = mutable_column->cloneEmpty();
|
||||
|
||||
|
||||
@ -186,6 +186,7 @@ void DatabaseLazy::attachTable(ContextPtr /* context_ */, const String & table_n
|
||||
throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists.", backQuote(database_name), backQuote(table_name));
|
||||
|
||||
it->second.expiration_iterator = cache_expiration_queue.emplace(cache_expiration_queue.end(), current_time, table_name);
|
||||
|
||||
CurrentMetrics::add(CurrentMetrics::AttachedTable, 1);
|
||||
}
|
||||
|
||||
@ -202,6 +203,7 @@ StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & ta
|
||||
if (it->second.expiration_iterator != cache_expiration_queue.end())
|
||||
cache_expiration_queue.erase(it->second.expiration_iterator);
|
||||
tables_cache.erase(it);
|
||||
|
||||
CurrentMetrics::sub(CurrentMetrics::AttachedTable, 1);
|
||||
}
|
||||
return res;
|
||||
|
||||
@ -73,9 +73,10 @@ zkutil::ZooKeeperPtr DatabaseReplicated::getZooKeeper() const
|
||||
return getContext()->getZooKeeper();
|
||||
}
|
||||
|
||||
static inline String getHostID(ContextPtr global_context, const UUID & db_uuid)
|
||||
static inline String getHostID(ContextPtr global_context, const UUID & db_uuid, bool secure)
|
||||
{
|
||||
return Cluster::Address::toString(getFQDNOrHostName(), global_context->getTCPPort()) + ':' + toString(db_uuid);
|
||||
UInt16 port = secure ? global_context->getTCPPortSecure().value_or(DBMS_DEFAULT_SECURE_PORT) : global_context->getTCPPort();
|
||||
return Cluster::Address::toString(getFQDNOrHostName(), port) + ':' + toString(db_uuid);
|
||||
}
|
||||
|
||||
static inline UInt64 getMetadataHash(const String & table_name, const String & metadata)
|
||||
@ -415,8 +416,10 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessL
|
||||
return;
|
||||
}
|
||||
|
||||
String host_id = getHostID(getContext(), db_uuid);
|
||||
if (is_create_query || replica_host_id != host_id)
|
||||
String host_id = getHostID(getContext(), db_uuid, cluster_auth_info.cluster_secure_connection);
|
||||
String host_id_default = getHostID(getContext(), db_uuid, false);
|
||||
|
||||
if (is_create_query || (replica_host_id != host_id && replica_host_id != host_id_default))
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::REPLICA_ALREADY_EXISTS,
|
||||
@ -424,6 +427,14 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessL
|
||||
replica_name, shard_name, zookeeper_path, replica_host_id, host_id);
|
||||
}
|
||||
|
||||
/// Before 24.6 we always created host_id with insecure port, even if cluster_auth_info.cluster_secure_connection was true.
|
||||
/// So not to break compatibility, we need to update host_id to secure one if cluster_auth_info.cluster_secure_connection is true.
|
||||
if (host_id != host_id_default && replica_host_id == host_id_default)
|
||||
{
|
||||
current_zookeeper->set(replica_path, host_id, -1);
|
||||
createEmptyLogEntry(current_zookeeper);
|
||||
}
|
||||
|
||||
/// Check that replica_group_name in ZooKeeper matches the local one and change it if necessary.
|
||||
String zk_replica_group_name;
|
||||
if (!current_zookeeper->tryGet(replica_path + "/replica_group", zk_replica_group_name))
|
||||
@ -550,7 +561,7 @@ void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPt
|
||||
"already contains some data and it does not look like Replicated database path.", zookeeper_path);
|
||||
|
||||
/// Write host name to replica_path, it will protect from multiple replicas with the same name
|
||||
auto host_id = getHostID(getContext(), db_uuid);
|
||||
auto host_id = getHostID(getContext(), db_uuid, cluster_auth_info.cluster_secure_connection);
|
||||
|
||||
for (int attempts = 10; attempts > 0; --attempts)
|
||||
{
|
||||
|
||||
@ -260,7 +260,9 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
|
||||
res = it->second;
|
||||
tables.erase(it);
|
||||
res->is_detached = true;
|
||||
CurrentMetrics::sub(getAttachedCounterForStorage(res), 1);
|
||||
|
||||
if (res->isSystemStorage() == false)
|
||||
CurrentMetrics::sub(getAttachedCounterForStorage(res), 1);
|
||||
|
||||
auto table_id = res->getStorageID();
|
||||
if (table_id.hasUUID())
|
||||
@ -301,7 +303,9 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
|
||||
/// It is important to reset is_detached here since in case of RENAME in
|
||||
/// non-Atomic database the is_detached is set to true before RENAME.
|
||||
table->is_detached = false;
|
||||
CurrentMetrics::add(getAttachedCounterForStorage(table), 1);
|
||||
|
||||
if (table->isSystemStorage() == false && table_id.database_name != DatabaseCatalog::SYSTEM_DATABASE)
|
||||
CurrentMetrics::add(getAttachedCounterForStorage(table), 1);
|
||||
}
|
||||
|
||||
void DatabaseWithOwnTablesBase::shutdown()
|
||||
|
||||
@ -175,8 +175,7 @@ Columns DirectDictionary<dictionary_key_type>::getColumns(
|
||||
if (!mask_filled)
|
||||
(*default_mask)[requested_key_index] = 1;
|
||||
|
||||
Field value{};
|
||||
result_column->insert(value);
|
||||
result_column->insertDefault();
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -181,6 +181,8 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
|
||||
format_settings.pretty.highlight_digit_groups = settings.output_format_pretty_highlight_digit_groups;
|
||||
format_settings.pretty.output_format_pretty_row_numbers = settings.output_format_pretty_row_numbers;
|
||||
format_settings.pretty.output_format_pretty_single_large_number_tip_threshold = settings.output_format_pretty_single_large_number_tip_threshold;
|
||||
format_settings.pretty.output_format_pretty_display_footer_column_names = settings.output_format_pretty_display_footer_column_names;
|
||||
format_settings.pretty.output_format_pretty_display_footer_column_names_min_rows = settings.output_format_pretty_display_footer_column_names_min_rows;
|
||||
format_settings.protobuf.input_flatten_google_wrappers = settings.input_format_protobuf_flatten_google_wrappers;
|
||||
format_settings.protobuf.output_nullables_with_google_wrappers = settings.output_format_protobuf_nullables_with_google_wrappers;
|
||||
format_settings.protobuf.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user