mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-26 01:22:04 +00:00
Merge branch 'master' of github.com:ClickHouse/ClickHouse into variant-new-serialization
This commit is contained in:
avogar
commit
8a0dd60650
3
.github/workflows/master.yml
vendored
3
.github/workflows/master.yml
vendored
@ -104,10 +104,9 @@ jobs:
|
||||
with:
|
||||
stage: Tests_2
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# stage for jobs that do not prohibit merge
|
||||
Tests_3:
|
||||
# Test_3 should not wait for Test_1/Test_2 and should not be blocked by them on master branch since all jobs need to run there.
|
||||
needs: [RunConfig, Builds_1, Builds_2]
|
||||
needs: [RunConfig, Builds_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
|
||||
uses: ./.github/workflows/reusable_test_stage.yml
|
||||
with:
|
||||
|
||||
9
.github/workflows/pull_request.yml
vendored
9
.github/workflows/pull_request.yml
vendored
@ -134,9 +134,8 @@ jobs:
|
||||
with:
|
||||
stage: Tests_2
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# stage for jobs that do not prohibit merge
|
||||
Tests_3:
|
||||
needs: [RunConfig, Builds_1, Tests_1, Builds_2, Tests_2]
|
||||
needs: [RunConfig, Builds_1, Tests_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
|
||||
uses: ./.github/workflows/reusable_test_stage.yml
|
||||
with:
|
||||
@ -157,7 +156,8 @@ jobs:
|
||||
|
||||
CheckReadyForMerge:
|
||||
if: ${{ !cancelled() && needs.StyleCheck.result == 'success' }}
|
||||
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2]
|
||||
# Test_2 or Test_3 must not have jobs required for Mergeable check
|
||||
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1]
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
@ -196,8 +196,7 @@ jobs:
|
||||
concurrency:
|
||||
group: jepsen
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse Keeper Jepsen') }}
|
||||
# jepsen needs binary_release build which is in Builds_2
|
||||
needs: [RunConfig, Builds_2]
|
||||
needs: [RunConfig, Builds_1]
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse Keeper Jepsen
|
||||
|
||||
@ -23,9 +23,6 @@
|
||||
#include <openssl/conf.h>
|
||||
#endif
|
||||
|
||||
#if __has_feature(address_sanitizer)
|
||||
#include <sanitizer/lsan_interface.h>
|
||||
#endif
|
||||
|
||||
using Poco::RandomInputStream;
|
||||
using Poco::Thread;
|
||||
@ -70,18 +67,12 @@ void OpenSSLInitializer::initialize()
|
||||
SSL_library_init();
|
||||
SSL_load_error_strings();
|
||||
OpenSSL_add_all_algorithms();
|
||||
|
||||
|
||||
char seed[SEEDSIZE];
|
||||
RandomInputStream rnd;
|
||||
rnd.read(seed, sizeof(seed));
|
||||
{
|
||||
# if __has_feature(address_sanitizer)
|
||||
/// Leak sanitizer (part of address sanitizer) thinks that a few bytes of memory in OpenSSL are allocated during but never released.
|
||||
__lsan::ScopedDisabler lsan_disabler;
|
||||
#endif
|
||||
RAND_seed(seed, SEEDSIZE);
|
||||
}
|
||||
|
||||
RAND_seed(seed, SEEDSIZE);
|
||||
|
||||
int nMutexes = CRYPTO_num_locks();
|
||||
_mutexes = new Poco::FastMutex[nMutexes];
|
||||
CRYPTO_set_locking_callback(&OpenSSLInitializer::lock);
|
||||
@ -89,8 +80,8 @@ void OpenSSLInitializer::initialize()
|
||||
// https://sourceforge.net/p/poco/bugs/110/
|
||||
//
|
||||
// From http://www.openssl.org/docs/crypto/threads.html :
|
||||
// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(),
|
||||
// then a default implementation is used - on Windows and BeOS this uses the system's
|
||||
// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(),
|
||||
// then a default implementation is used - on Windows and BeOS this uses the system's
|
||||
// default thread identifying APIs"
|
||||
CRYPTO_set_id_callback(&OpenSSLInitializer::id);
|
||||
CRYPTO_set_dynlock_create_callback(&OpenSSLInitializer::dynlockCreate);
|
||||
@ -109,7 +100,7 @@ void OpenSSLInitializer::uninitialize()
|
||||
CRYPTO_set_locking_callback(0);
|
||||
CRYPTO_set_id_callback(0);
|
||||
delete [] _mutexes;
|
||||
|
||||
|
||||
CONF_modules_free();
|
||||
}
|
||||
}
|
||||
|
||||
@ -213,6 +213,7 @@ target_compile_definitions (_poco_foundation
|
||||
)
|
||||
|
||||
target_include_directories (_poco_foundation SYSTEM PUBLIC "include")
|
||||
target_link_libraries (_poco_foundation PRIVATE clickhouse_common_io)
|
||||
|
||||
target_link_libraries (_poco_foundation
|
||||
PRIVATE
|
||||
|
||||
@ -48,7 +48,13 @@ class Foundation_API ThreadPool
|
||||
/// from the pool.
|
||||
{
|
||||
public:
|
||||
ThreadPool(int minCapacity = 2, int maxCapacity = 16, int idleTime = 60, int stackSize = POCO_THREAD_STACK_SIZE);
|
||||
explicit ThreadPool(
|
||||
int minCapacity = 2,
|
||||
int maxCapacity = 16,
|
||||
int idleTime = 60,
|
||||
int stackSize = POCO_THREAD_STACK_SIZE,
|
||||
size_t global_profiler_real_time_period_ns_ = 0,
|
||||
size_t global_profiler_cpu_time_period_ns_ = 0);
|
||||
/// Creates a thread pool with minCapacity threads.
|
||||
/// If required, up to maxCapacity threads are created
|
||||
/// a NoThreadAvailableException exception is thrown.
|
||||
@ -56,8 +62,14 @@ public:
|
||||
/// and more than minCapacity threads are running, the thread
|
||||
/// is killed. Threads are created with given stack size.
|
||||
|
||||
ThreadPool(
|
||||
const std::string & name, int minCapacity = 2, int maxCapacity = 16, int idleTime = 60, int stackSize = POCO_THREAD_STACK_SIZE);
|
||||
explicit ThreadPool(
|
||||
const std::string & name,
|
||||
int minCapacity = 2,
|
||||
int maxCapacity = 16,
|
||||
int idleTime = 60,
|
||||
int stackSize = POCO_THREAD_STACK_SIZE,
|
||||
size_t global_profiler_real_time_period_ns_ = 0,
|
||||
size_t global_profiler_cpu_time_period_ns_ = 0);
|
||||
/// Creates a thread pool with the given name and minCapacity threads.
|
||||
/// If required, up to maxCapacity threads are created
|
||||
/// a NoThreadAvailableException exception is thrown.
|
||||
@ -171,6 +183,8 @@ private:
|
||||
int _serial;
|
||||
int _age;
|
||||
int _stackSize;
|
||||
size_t _globalProfilerRealTimePeriodNs;
|
||||
size_t _globalProfilerCPUTimePeriodNs;
|
||||
ThreadVec _threads;
|
||||
mutable FastMutex _mutex;
|
||||
};
|
||||
|
||||
@ -20,6 +20,7 @@
|
||||
#include "Poco/ErrorHandler.h"
|
||||
#include <sstream>
|
||||
#include <ctime>
|
||||
#include <Common/ThreadPool.h>
|
||||
|
||||
|
||||
namespace Poco {
|
||||
@ -28,7 +29,11 @@ namespace Poco {
|
||||
class PooledThread: public Runnable
|
||||
{
|
||||
public:
|
||||
PooledThread(const std::string& name, int stackSize = POCO_THREAD_STACK_SIZE);
|
||||
explicit PooledThread(
|
||||
const std::string& name,
|
||||
int stackSize = POCO_THREAD_STACK_SIZE,
|
||||
size_t globalProfilerRealTimePeriodNs_ = 0,
|
||||
size_t globalProfilerCPUTimePeriodNs_ = 0);
|
||||
~PooledThread();
|
||||
|
||||
void start();
|
||||
@ -51,16 +56,24 @@ private:
|
||||
Event _targetCompleted;
|
||||
Event _started;
|
||||
FastMutex _mutex;
|
||||
size_t _globalProfilerRealTimePeriodNs;
|
||||
size_t _globalProfilerCPUTimePeriodNs;
|
||||
};
|
||||
|
||||
|
||||
PooledThread::PooledThread(const std::string& name, int stackSize):
|
||||
_idle(true),
|
||||
_idleTime(0),
|
||||
_pTarget(0),
|
||||
_name(name),
|
||||
PooledThread::PooledThread(
|
||||
const std::string& name,
|
||||
int stackSize,
|
||||
size_t globalProfilerRealTimePeriodNs_,
|
||||
size_t globalProfilerCPUTimePeriodNs_) :
|
||||
_idle(true),
|
||||
_idleTime(0),
|
||||
_pTarget(0),
|
||||
_name(name),
|
||||
_thread(name),
|
||||
_targetCompleted(false)
|
||||
_targetCompleted(false),
|
||||
_globalProfilerRealTimePeriodNs(globalProfilerRealTimePeriodNs_),
|
||||
_globalProfilerCPUTimePeriodNs(globalProfilerCPUTimePeriodNs_)
|
||||
{
|
||||
poco_assert_dbg (stackSize >= 0);
|
||||
_thread.setStackSize(stackSize);
|
||||
@ -83,7 +96,7 @@ void PooledThread::start()
|
||||
void PooledThread::start(Thread::Priority priority, Runnable& target)
|
||||
{
|
||||
FastMutex::ScopedLock lock(_mutex);
|
||||
|
||||
|
||||
poco_assert (_pTarget == 0);
|
||||
|
||||
_pTarget = ⌖
|
||||
@ -109,7 +122,7 @@ void PooledThread::start(Thread::Priority priority, Runnable& target, const std:
|
||||
}
|
||||
_thread.setName(fullName);
|
||||
_thread.setPriority(priority);
|
||||
|
||||
|
||||
poco_assert (_pTarget == 0);
|
||||
|
||||
_pTarget = ⌖
|
||||
@ -145,7 +158,7 @@ void PooledThread::join()
|
||||
void PooledThread::activate()
|
||||
{
|
||||
FastMutex::ScopedLock lock(_mutex);
|
||||
|
||||
|
||||
poco_assert (_idle);
|
||||
_idle = false;
|
||||
_targetCompleted.reset();
|
||||
@ -155,7 +168,7 @@ void PooledThread::activate()
|
||||
void PooledThread::release()
|
||||
{
|
||||
const long JOIN_TIMEOUT = 10000;
|
||||
|
||||
|
||||
_mutex.lock();
|
||||
_pTarget = 0;
|
||||
_mutex.unlock();
|
||||
@ -174,6 +187,10 @@ void PooledThread::release()
|
||||
|
||||
void PooledThread::run()
|
||||
{
|
||||
DB::ThreadStatus thread_status;
|
||||
if (unlikely(_globalProfilerRealTimePeriodNs != 0 || _globalProfilerCPUTimePeriodNs != 0))
|
||||
thread_status.initGlobalProfiler(_globalProfilerRealTimePeriodNs, _globalProfilerCPUTimePeriodNs);
|
||||
|
||||
_started.set();
|
||||
for (;;)
|
||||
{
|
||||
@ -220,13 +237,17 @@ void PooledThread::run()
|
||||
ThreadPool::ThreadPool(int minCapacity,
|
||||
int maxCapacity,
|
||||
int idleTime,
|
||||
int stackSize):
|
||||
_minCapacity(minCapacity),
|
||||
_maxCapacity(maxCapacity),
|
||||
int stackSize,
|
||||
size_t globalProfilerRealTimePeriodNs_,
|
||||
size_t globalProfilerCPUTimePeriodNs_) :
|
||||
_minCapacity(minCapacity),
|
||||
_maxCapacity(maxCapacity),
|
||||
_idleTime(idleTime),
|
||||
_serial(0),
|
||||
_age(0),
|
||||
_stackSize(stackSize)
|
||||
_stackSize(stackSize),
|
||||
_globalProfilerRealTimePeriodNs(globalProfilerRealTimePeriodNs_),
|
||||
_globalProfilerCPUTimePeriodNs(globalProfilerCPUTimePeriodNs_)
|
||||
{
|
||||
poco_assert (minCapacity >= 1 && maxCapacity >= minCapacity && idleTime > 0);
|
||||
|
||||
@ -243,14 +264,18 @@ ThreadPool::ThreadPool(const std::string& name,
|
||||
int minCapacity,
|
||||
int maxCapacity,
|
||||
int idleTime,
|
||||
int stackSize):
|
||||
int stackSize,
|
||||
size_t globalProfilerRealTimePeriodNs_,
|
||||
size_t globalProfilerCPUTimePeriodNs_) :
|
||||
_name(name),
|
||||
_minCapacity(minCapacity),
|
||||
_maxCapacity(maxCapacity),
|
||||
_minCapacity(minCapacity),
|
||||
_maxCapacity(maxCapacity),
|
||||
_idleTime(idleTime),
|
||||
_serial(0),
|
||||
_age(0),
|
||||
_stackSize(stackSize)
|
||||
_stackSize(stackSize),
|
||||
_globalProfilerRealTimePeriodNs(globalProfilerRealTimePeriodNs_),
|
||||
_globalProfilerCPUTimePeriodNs(globalProfilerCPUTimePeriodNs_)
|
||||
{
|
||||
poco_assert (minCapacity >= 1 && maxCapacity >= minCapacity && idleTime > 0);
|
||||
|
||||
@ -393,15 +418,15 @@ void ThreadPool::housekeep()
|
||||
ThreadVec activeThreads;
|
||||
idleThreads.reserve(_threads.size());
|
||||
activeThreads.reserve(_threads.size());
|
||||
|
||||
|
||||
for (ThreadVec::iterator it = _threads.begin(); it != _threads.end(); ++it)
|
||||
{
|
||||
if ((*it)->idle())
|
||||
{
|
||||
if ((*it)->idleTime() < _idleTime)
|
||||
idleThreads.push_back(*it);
|
||||
else
|
||||
expiredThreads.push_back(*it);
|
||||
else
|
||||
expiredThreads.push_back(*it);
|
||||
}
|
||||
else activeThreads.push_back(*it);
|
||||
}
|
||||
@ -463,7 +488,7 @@ PooledThread* ThreadPool::createThread()
|
||||
{
|
||||
std::ostringstream name;
|
||||
name << _name << "[#" << ++_serial << "]";
|
||||
return new PooledThread(name.str(), _stackSize);
|
||||
return new PooledThread(name.str(), _stackSize, _globalProfilerRealTimePeriodNs, _globalProfilerCPUTimePeriodNs);
|
||||
}
|
||||
|
||||
|
||||
@ -481,7 +506,7 @@ public:
|
||||
ThreadPool* pool()
|
||||
{
|
||||
FastMutex::ScopedLock lock(_mutex);
|
||||
|
||||
|
||||
if (!_pPool)
|
||||
{
|
||||
_pPool = new ThreadPool("default");
|
||||
@ -490,7 +515,7 @@ public:
|
||||
}
|
||||
return _pPool;
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
ThreadPool* _pPool;
|
||||
FastMutex _mutex;
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
# This variables autochanged by tests/ci/version_helper.py:
|
||||
|
||||
# NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
|
||||
# NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION,
|
||||
# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
|
||||
SET(VERSION_REVISION 54487)
|
||||
SET(VERSION_REVISION 54488)
|
||||
SET(VERSION_MAJOR 24)
|
||||
SET(VERSION_MINOR 6)
|
||||
SET(VERSION_MINOR 7)
|
||||
SET(VERSION_PATCH 1)
|
||||
SET(VERSION_GITHASH 70a1d3a63d47f0be077d67b8deb907230fc7cfb0)
|
||||
SET(VERSION_DESCRIBE v24.6.1.1-testing)
|
||||
SET(VERSION_STRING 24.6.1.1)
|
||||
SET(VERSION_GITHASH aa023477a9265e403982fca5ee29a714db5133d9)
|
||||
SET(VERSION_DESCRIBE v24.7.1.1-testing)
|
||||
SET(VERSION_STRING 24.7.1.1)
|
||||
# end of autochange
|
||||
|
||||
2
contrib/openssl
vendored
2
contrib/openssl
vendored
@ -1 +1 @@
|
||||
Subproject commit e0d6ae2bf93cf6dc26bb86aa39992bc6a410869a
|
||||
Subproject commit 277de2ba202af4eb2291b363456d32ff0960e559
|
||||
@ -37,7 +37,7 @@ Using named collections:
|
||||
<named_collections>
|
||||
<iceberg_conf>
|
||||
<url>http://test.s3.amazonaws.com/clickhouse-bucket/</url>
|
||||
<access_key_id>test<access_key_id>
|
||||
<access_key_id>test</access_key_id>
|
||||
<secret_access_key>test</secret_access_key>
|
||||
</iceberg_conf>
|
||||
</named_collections>
|
||||
|
||||
@ -13,7 +13,7 @@ This engine provides integration with [Amazon S3](https://aws.amazon.com/s3/) ec
|
||||
CREATE TABLE s3_queue_engine_table (name String, value UInt32)
|
||||
ENGINE = S3Queue(path, [NOSIGN, | aws_access_key_id, aws_secret_access_key,] format, [compression])
|
||||
[SETTINGS]
|
||||
[mode = 'unordered',]
|
||||
[mode = '',]
|
||||
[after_processing = 'keep',]
|
||||
[keeper_path = '',]
|
||||
[s3queue_loading_retries = 0,]
|
||||
|
||||
@ -31,6 +31,56 @@ Alternatively, in order to enable the MySQL interface for an existing service:
|
||||
3. After entering the password, you will get prompted the MySQL connection string for this service
|
||||
|
||||
|
||||
## Creating multiple MySQL users in ClickHouse Cloud
|
||||
|
||||
By default, there is a built-in `mysql4<subdomain>` user, which uses the same password as the `default` one. The `<subdomain>` part is the first segment of your ClickHouse Cloud hostname. This format is necessary to work with the tools that implement secure connection, but don't provide [SNI information in their TLS handshake](https://www.cloudflare.com/learning/ssl/what-is-sni), which makes it impossible to do the internal routing without an extra hint in the username (MySQL console client is one of such tools).
|
||||
|
||||
Because of this, we _highly recommend_ following the `mysql4<subdomain>_<username>` format when creating a new user intended to be used with the MySQL interface, where `<subdomain>` is a hint to identify your Cloud service, and `<username>` is an arbitrary suffix of your choice.
|
||||
|
||||
:::tip
|
||||
For ClickHouse Cloud hostname like `foobar.us-east1.aws.clickhouse.cloud`, the `<subdomain>` part equals to `foobar`, and a custom MySQL username could look like `mysql4foobar_team1`.
|
||||
:::
|
||||
|
||||
You can create extra users to use with the MySQL interface if, for example, you need to apply extra settings.
|
||||
|
||||
1. Optional - create a [settings profile](https://clickhouse.com/docs/en/sql-reference/statements/create/settings-profile) to apply for your custom user. For example, `my_custom_profile` with an extra setting which will be applied by default when we connect with the user we create later:
|
||||
|
||||
```sql
|
||||
CREATE SETTINGS PROFILE my_custom_profile SETTINGS prefer_column_name_to_alias=1;
|
||||
```
|
||||
|
||||
`prefer_column_name_to_alias` is used just as an example, you can use other settings there.
|
||||
2. [Create a user](https://clickhouse.com/docs/en/sql-reference/statements/create/user) using the following format: `mysql4<subdomain>_<username>` ([see above](#creating-multiple-mysql-users-in-clickhouse-cloud)). The password must be in double SHA1 format. For example:
|
||||
|
||||
```sql
|
||||
CREATE USER mysql4foobar_team1 IDENTIFIED WITH double_sha1_password BY 'YourPassword42$';
|
||||
```
|
||||
|
||||
or if you want to use a custom profile for this user:
|
||||
|
||||
```sql
|
||||
CREATE USER mysql4foobar_team1 IDENTIFIED WITH double_sha1_password BY 'YourPassword42$' SETTINGS PROFILE 'my_custom_profile';
|
||||
```
|
||||
|
||||
where `my_custom_profile` is the name of the profile you created earlier.
|
||||
3. [Grant](https://clickhouse.com/docs/en/sql-reference/statements/grant) the new user the necessary permissions to interact with the desired tables or databases. For example, if you want to grant access to `system.query_log` only:
|
||||
|
||||
```sql
|
||||
GRANT SELECT ON system.query_log TO mysql4foobar_team1;
|
||||
```
|
||||
|
||||
4. Use the created user to connect to your ClickHouse Cloud service with the MySQL interface.
|
||||
|
||||
### Troubleshooting multiple MySQL users in ClickHouse Cloud
|
||||
|
||||
If you created a new MySQL user, and you see the following error while connecting via MySQL CLI client:
|
||||
|
||||
```
|
||||
ERROR 2013 (HY000): Lost connection to MySQL server at 'reading authorization packet', system error: 54
|
||||
```
|
||||
|
||||
In this case, ensure that the username follows the `mysql4<subdomain>_<username>` format, as described ([above](#creating-multiple-mysql-users-in-clickhouse-cloud)).
|
||||
|
||||
## Enabling the MySQL Interface On Self-managed ClickHouse
|
||||
|
||||
Add the [mysql_port](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-mysql_port) setting to your server's configuration file. For example, you could define the port in a new XML file in your `config.d/` [folder](../operations/configuration-files):
|
||||
|
||||
@ -591,6 +591,22 @@ Default value: 100000
|
||||
<max_part_num_to_warn>400</max_part_num_to_warn>
|
||||
```
|
||||
|
||||
## max\_table\_num\_to\_throw {#max-table-num-to-throw}
|
||||
If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.Default value: 0
|
||||
|
||||
**Example**
|
||||
```xml
|
||||
<max_table_num_to_throw>400</max_table_num_to_throw>
|
||||
```
|
||||
|
||||
## max\_database\_num\_to\_throw {#max-table-num-to-throw}
|
||||
If number of _database is greater than this value, server will throw an exception. 0 means no limitation.
|
||||
Default value: 0
|
||||
|
||||
**Example**
|
||||
```xml
|
||||
<max_database_num_to_throw>400</max_database_num_to_throw>
|
||||
```
|
||||
|
||||
## max_temporary_data_on_disk_size
|
||||
|
||||
|
||||
@ -1592,19 +1592,19 @@ Default value: `default`.
|
||||
|
||||
## parallel_replicas_custom_key_range_lower {#parallel_replicas_custom_key_range_lower}
|
||||
|
||||
Allows the filter type `range` to split the work evenly between replicas based on the custom range `[parallel_replicas_custom_key_range_lower, INT_MAX]`.
|
||||
Allows the filter type `range` to split the work evenly between replicas based on the custom range `[parallel_replicas_custom_key_range_lower, INT_MAX]`.
|
||||
|
||||
When used in conjuction with [parallel_replicas_custom_key_range_upper](#parallel_replicas_custom_key_range_upper), it lets the filter evenly split the work over replicas for the range `[parallel_replicas_custom_key_range_lower, parallel_replicas_custom_key_range_upper]`.
|
||||
When used in conjuction with [parallel_replicas_custom_key_range_upper](#parallel_replicas_custom_key_range_upper), it lets the filter evenly split the work over replicas for the range `[parallel_replicas_custom_key_range_lower, parallel_replicas_custom_key_range_upper]`.
|
||||
|
||||
Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing.
|
||||
Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing.
|
||||
|
||||
## parallel_replicas_custom_key_range_upper {#parallel_replicas_custom_key_range_upper}
|
||||
|
||||
Allows the filter type `range` to split the work evenly between replicas based on the custom range `[0, parallel_replicas_custom_key_range_upper]`. A value of 0 disables the upper bound, setting it the max value of the custom key expression.
|
||||
|
||||
When used in conjuction with [parallel_replicas_custom_key_range_lower](#parallel_replicas_custom_key_range_lower), it lets the filter evenly split the work over replicas for the range `[parallel_replicas_custom_key_range_lower, parallel_replicas_custom_key_range_upper]`.
|
||||
When used in conjuction with [parallel_replicas_custom_key_range_lower](#parallel_replicas_custom_key_range_lower), it lets the filter evenly split the work over replicas for the range `[parallel_replicas_custom_key_range_lower, parallel_replicas_custom_key_range_upper]`.
|
||||
|
||||
Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing.
|
||||
Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing.
|
||||
|
||||
## allow_experimental_parallel_reading_from_replicas
|
||||
|
||||
@ -3188,7 +3188,7 @@ Default value: `0`.
|
||||
|
||||
## lightweight_deletes_sync {#lightweight_deletes_sync}
|
||||
|
||||
The same as 'mutation_sync', but controls only execution of lightweight deletes.
|
||||
The same as 'mutation_sync', but controls only execution of lightweight deletes.
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -5150,7 +5150,7 @@ Allows using statistic to optimize the order of [prewhere conditions](../../sql-
|
||||
|
||||
## analyze_index_with_space_filling_curves
|
||||
|
||||
If a table has a space-filling curve in its index, e.g. `ORDER BY mortonEncode(x, y)`, and the query has conditions on its arguments, e.g. `x >= 10 AND x <= 20 AND y >= 20 AND y <= 30`, use the space-filling curve for index analysis.
|
||||
If a table has a space-filling curve in its index, e.g. `ORDER BY mortonEncode(x, y)` or `ORDER BY hilbertEncode(x, y)`, and the query has conditions on its arguments, e.g. `x >= 10 AND x <= 20 AND y >= 20 AND y <= 30`, use the space-filling curve for index analysis.
|
||||
|
||||
## query_plan_enable_optimizations {#query_plan_enable_optimizations}
|
||||
|
||||
@ -5418,6 +5418,12 @@ When set to `false` than all attempts are made with identical timeouts.
|
||||
|
||||
Default value: `true`.
|
||||
|
||||
## uniform_snowflake_conversion_functions {#uniform_snowflake_conversion_functions}
|
||||
|
||||
If set to `true`, then functions `snowflakeIDToDateTime`, `snowflakeIDToDateTime64`, `dateTimeToSnowflakeID`, and `dateTime64ToSnowflakeID` are enabled, and functions `snowflakeToDateTime`, `snowflakeToDateTime64`, `dateTimeToSnowflake`, and `dateTime64ToSnowflake` are disabled (and vice versa if set to `false`).
|
||||
|
||||
Default value: `true`
|
||||
|
||||
## allow_experimental_variant_type {#allow_experimental_variant_type}
|
||||
|
||||
Allows creation of experimental [Variant](../../sql-reference/data-types/variant.md).
|
||||
|
||||
@ -113,6 +113,8 @@ Columns:
|
||||
- `used_functions` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `functions`, which were used during query execution.
|
||||
- `used_storages` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `storages`, which were used during query execution.
|
||||
- `used_table_functions` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `table functions`, which were used during query execution.
|
||||
- `used_privileges` ([Array(String)](../../sql-reference/data-types/array.md)) - Privileges which were successfully checked during query execution.
|
||||
- `missing_privileges` ([Array(String)](../../sql-reference/data-types/array.md)) - Privileges that are missing during query execution.
|
||||
- `query_cache_usage` ([Enum8](../../sql-reference/data-types/enum.md)) — Usage of the [query cache](../query-cache.md) during query execution. Values:
|
||||
- `'Unknown'` = Status unknown.
|
||||
- `'None'` = The query result was neither written into nor read from the query cache.
|
||||
@ -194,6 +196,8 @@ used_formats: []
|
||||
used_functions: []
|
||||
used_storages: []
|
||||
used_table_functions: []
|
||||
used_privileges: []
|
||||
missing_privileges: []
|
||||
query_cache_usage: None
|
||||
```
|
||||
|
||||
|
||||
@ -1,90 +0,0 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/groupconcat
|
||||
sidebar_position: 363
|
||||
sidebar_label: groupConcat
|
||||
title: groupConcat
|
||||
---
|
||||
|
||||
Calculates a concatenated string from a group of strings, optionally separated by a delimiter, and optionally limited by a maximum number of elements.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
groupConcat(expression [, delimiter] [, limit]);
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `expression` — The expression or column name that outputs strings to be concatenated..
|
||||
- `delimiter` — A [string](../../../sql-reference/data-types/string.md) that will be used to separate concatenated values. This parameter is optional and defaults to an empty string if not specified.
|
||||
- `limit` — A positive [integer](../../../sql-reference/data-types/int-uint.md) specifying the maximum number of elements to concatenate. If more elements are present, excess elements are ignored. This parameter is optional.
|
||||
|
||||
:::note
|
||||
If delimiter is specified without limit, it must be the first parameter following the expression. If both delimiter and limit are specified, delimiter must precede limit.
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Returns a [string](../../../sql-reference/data-types/string.md) consisting of the concatenated values of the column or expression. If the group has no elements or only null elements, and the function does not specify a handling for only null values, the result is a nullable string with a null value.
|
||||
|
||||
**Examples**
|
||||
|
||||
Input table:
|
||||
|
||||
``` text
|
||||
┌─id─┬─name─┐
|
||||
│ 1 │ John│
|
||||
│ 2 │ Jane│
|
||||
│ 3 │ Bob│
|
||||
└────┴──────┘
|
||||
```
|
||||
|
||||
1. Basic usage without a delimiter:
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT groupConcat(Name) FROM Employees;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
JohnJaneBob
|
||||
```
|
||||
|
||||
This concatenates all names into one continuous string without any separator.
|
||||
|
||||
|
||||
2. Using comma as a delimiter:
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT groupConcat(Name, ', ', 2) FROM Employees;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
John, Jane, Bob
|
||||
```
|
||||
|
||||
This output shows the names separated by a comma followed by a space.
|
||||
|
||||
|
||||
3. Limiting the number of concatenated elements
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT groupConcat(Name, ', ', 2) FROM Employees;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
John, Jane
|
||||
```
|
||||
|
||||
This query limits the output to the first two names, even though there are more names in the table.
|
||||
@ -137,7 +137,7 @@ If the time transition (due to daylight saving time or for other reasons) was pe
|
||||
|
||||
Non-monotonic calendar dates. For example, in Happy Valley - Goose Bay, the time was transitioned one hour backwards at 00:01:00 7 Nov 2010 (one minute after midnight). So after 6th Nov has ended, people observed a whole one minute of 7th Nov, then time was changed back to 23:01 6th Nov and after another 59 minutes the 7th Nov started again. ClickHouse does not (yet) support this kind of fun. During these days the results of time processing functions may be slightly incorrect.
|
||||
|
||||
Similar issue exists for Casey Antarctic station in year 2010. They changed time three hours back at 5 Mar, 02:00. If you are working in antarctic station, please don't afraid to use ClickHouse. Just make sure you set timezone to UTC or be aware of inaccuracies.
|
||||
Similar issue exists for Casey Antarctic station in year 2010. They changed time three hours back at 5 Mar, 02:00. If you are working in antarctic station, please don't be afraid to use ClickHouse. Just make sure you set timezone to UTC or be aware of inaccuracies.
|
||||
|
||||
Time shifts for multiple days. Some pacific islands changed their timezone offset from UTC+14 to UTC-12. That's alright but some inaccuracies may present if you do calculations with their timezone for historical time points at the days of conversion.
|
||||
|
||||
|
||||
@ -543,12 +543,17 @@ serverUUID()
|
||||
|
||||
Generates a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID).
|
||||
|
||||
The generated Snowflake ID contains the current Unix timestamp in milliseconds 41 (+ 1 top zero bit) bits, followed by machine id (10 bits), a counter (12 bits) to distinguish IDs within a millisecond.
|
||||
The generated Snowflake ID contains the current Unix timestamp in milliseconds (41 + 1 top zero bits), followed by a machine id (10 bits), and a counter (12 bits) to distinguish IDs within a millisecond.
|
||||
For any given timestamp (unix_ts_ms), the counter starts at 0 and is incremented by 1 for each new Snowflake ID until the timestamp changes.
|
||||
In case the counter overflows, the timestamp field is incremented by 1 and the counter is reset to 0.
|
||||
|
||||
Function `generateSnowflakeID` guarantees that the counter field within a timestamp increments monotonically across all function invocations in concurrently running threads and queries.
|
||||
|
||||
:::note
|
||||
The generated Snowflake IDs are based on the UNIX epoch 1970-01-01.
|
||||
While no standard or recommendation exists for the epoch of Snowflake IDs, implementations in other systems may use a different epoch, e.g. Twitter/X (2010-11-04) or Mastodon (2015-01-01).
|
||||
:::
|
||||
|
||||
```
|
||||
0 1 2 3
|
||||
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
|
||||
@ -605,6 +610,11 @@ SELECT generateSnowflakeID(1), generateSnowflakeID(2);
|
||||
|
||||
## snowflakeToDateTime
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Extracts the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) in [DateTime](../data-types/datetime.md) format.
|
||||
|
||||
**Syntax**
|
||||
@ -641,6 +651,11 @@ Result:
|
||||
|
||||
## snowflakeToDateTime64
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Extracts the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) in [DateTime64](../data-types/datetime64.md) format.
|
||||
|
||||
**Syntax**
|
||||
@ -677,6 +692,11 @@ Result:
|
||||
|
||||
## dateTimeToSnowflake
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Converts a [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
@ -711,6 +731,11 @@ Result:
|
||||
|
||||
## dateTime64ToSnowflake
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Convert a [DateTime64](../data-types/datetime64.md) to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
@ -743,6 +768,148 @@ Result:
|
||||
└─────────────────────────────┘
|
||||
```
|
||||
|
||||
## snowflakeIDToDateTime
|
||||
|
||||
Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime](../data-types/datetime.md).
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
snowflakeIDToDateTime(value[, epoch[, time_zone]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Snowflake ID. [UInt64](../data-types/int-uint.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
- `time_zone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The timestamp component of `value` as a [DateTime](../data-types/datetime.md) value.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT snowflakeIDToDateTime(7204436857747984384) AS res
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌─────────────────res─┐
|
||||
│ 2024-06-06 10:59:58 │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## snowflakeIDToDateTime64
|
||||
|
||||
Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime64](../data-types/datetime64.md).
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
snowflakeIDToDateTime64(value[, epoch[, time_zone]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Snowflake ID. [UInt64](../data-types/int-uint.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
- `time_zone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The timestamp component of `value` as a [DateTime64](../data-types/datetime64.md) with scale = 3, i.e. millisecond precision.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT snowflakeIDToDateTime64(7204436857747984384) AS res
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌─────────────────res─┐
|
||||
│ 2024-06-06 10:59:58 │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## dateTimeToSnowflakeID
|
||||
|
||||
Converts a [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
dateTimeToSnowflakeID(value[, epoch])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Date with time. [DateTime](../data-types/datetime.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai') AS dt, dateTimeToSnowflakeID(dt) AS res;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌──────────────────dt─┬─────────────────res─┐
|
||||
│ 2021-08-15 18:57:56 │ 6832626392367104000 │
|
||||
└─────────────────────┴─────────────────────┘
|
||||
```
|
||||
|
||||
## dateTime64ToSnowflakeID
|
||||
|
||||
Convert a [DateTime64](../data-types/datetime64.md) to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
dateTime64ToSnowflakeID(value[, epoch])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Date with time. [DateTime64](../data-types/datetime64.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT toDateTime('2021-08-15 18:57:56.493', 3, 'Asia/Shanghai') AS dt, dateTime64ToSnowflakeID(dt) AS res;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌──────────────────────dt─┬─────────────────res─┐
|
||||
│ 2021-08-15 18:57:56.493 │ 6832626394434895872 │
|
||||
└─────────────────────────┴─────────────────────┘
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [dictGetUUID](../functions/ext-dict-functions.md#ext_dict_functions-other)
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
#include <Poco/Net/NetException.h>
|
||||
#include <Poco/Util/HelpFormatter.h>
|
||||
#include <Poco/Environment.h>
|
||||
#include <Poco/Config.h>
|
||||
#include <Common/scope_guard_safe.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <base/phdr_cache.h>
|
||||
@ -721,11 +722,6 @@ try
|
||||
CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::getVersionRevision());
|
||||
CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
|
||||
|
||||
Poco::ThreadPool server_pool(3, server_settings.max_connections);
|
||||
std::mutex servers_lock;
|
||||
std::vector<ProtocolServerAdapter> servers;
|
||||
std::vector<ProtocolServerAdapter> servers_to_start_before_tables;
|
||||
|
||||
/** Context contains all that query execution is dependent:
|
||||
* settings, available functions, data types, aggregate functions, databases, ...
|
||||
*/
|
||||
@ -823,6 +819,18 @@ try
|
||||
total_memory_tracker.setSampleMaxAllocationSize(server_settings.total_memory_profiler_sample_max_allocation_size);
|
||||
}
|
||||
|
||||
Poco::ThreadPool server_pool(
|
||||
/* minCapacity */3,
|
||||
/* maxCapacity */server_settings.max_connections,
|
||||
/* idleTime */60,
|
||||
/* stackSize */POCO_THREAD_STACK_SIZE,
|
||||
server_settings.global_profiler_real_time_period_ns,
|
||||
server_settings.global_profiler_cpu_time_period_ns);
|
||||
|
||||
std::mutex servers_lock;
|
||||
std::vector<ProtocolServerAdapter> servers;
|
||||
std::vector<ProtocolServerAdapter> servers_to_start_before_tables;
|
||||
|
||||
/// Wait for all threads to avoid possible use-after-free (for example logging objects can be already destroyed).
|
||||
SCOPE_EXIT({
|
||||
Stopwatch watch;
|
||||
|
||||
@ -4,12 +4,12 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, AccessFlags access_flags_)
|
||||
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccessWrapper> & access_, AccessFlags access_flags_)
|
||||
: CachedAccessChecking(access_, AccessRightsElement{access_flags_})
|
||||
{
|
||||
}
|
||||
|
||||
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, const AccessRightsElement & element_)
|
||||
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccessWrapper> & access_, const AccessRightsElement & element_)
|
||||
: access(access_), element(element_)
|
||||
{
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Access/Common/AccessRightsElement.h>
|
||||
#include <Access/ContextAccess.h>
|
||||
#include <memory>
|
||||
|
||||
|
||||
@ -13,14 +14,14 @@ class ContextAccess;
|
||||
class CachedAccessChecking
|
||||
{
|
||||
public:
|
||||
CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, AccessFlags access_flags_);
|
||||
CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, const AccessRightsElement & element_);
|
||||
CachedAccessChecking(const std::shared_ptr<const ContextAccessWrapper> & access_, AccessFlags access_flags_);
|
||||
CachedAccessChecking(const std::shared_ptr<const ContextAccessWrapper> & access_, const AccessRightsElement & element_);
|
||||
~CachedAccessChecking();
|
||||
|
||||
bool checkAccess(bool throw_if_denied = true);
|
||||
|
||||
private:
|
||||
const std::shared_ptr<const ContextAccess> access;
|
||||
const std::shared_ptr<const ContextAccessWrapper> access;
|
||||
const AccessRightsElement element;
|
||||
bool checked = false;
|
||||
bool result = false;
|
||||
|
||||
@ -20,6 +20,7 @@
|
||||
#include <boost/algorithm/string/join.hpp>
|
||||
#include <boost/range/algorithm/set_algorithm.hpp>
|
||||
#include <cassert>
|
||||
#include <unordered_set>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -271,7 +272,7 @@ namespace
|
||||
|
||||
std::shared_ptr<const ContextAccess> ContextAccess::fromContext(const ContextPtr & context)
|
||||
{
|
||||
return context->getAccess();
|
||||
return ContextAccessWrapper::fromContext(context)->getAccess();
|
||||
}
|
||||
|
||||
|
||||
@ -560,7 +561,7 @@ std::shared_ptr<const AccessRights> ContextAccess::getAccessRightsWithImplicit()
|
||||
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... args) const
|
||||
bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, AccessFlags flags, const Args &... args) const
|
||||
{
|
||||
if (user_was_dropped)
|
||||
{
|
||||
@ -573,8 +574,10 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
if (params.full_access)
|
||||
return true;
|
||||
|
||||
auto access_granted = []
|
||||
auto access_granted = [&]
|
||||
{
|
||||
if constexpr (throw_if_denied)
|
||||
context->addQueryPrivilegesInfo(AccessRightsElement{flags, args...}.toStringWithoutOptions(), true);
|
||||
return true;
|
||||
};
|
||||
|
||||
@ -583,7 +586,10 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
FmtArgs && ...fmt_args [[maybe_unused]])
|
||||
{
|
||||
if constexpr (throw_if_denied)
|
||||
{
|
||||
context->addQueryPrivilegesInfo(AccessRightsElement{flags, args...}.toStringWithoutOptions(), false);
|
||||
throw Exception(error_code, std::move(fmt_string), getUserName(), std::forward<FmtArgs>(fmt_args)...);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
@ -686,102 +692,102 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool ContextAccess::checkAccessImpl(const AccessFlags & flags) const
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessFlags & flags) const
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(flags);
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(context, flags);
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool ContextAccess::checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessFlags & flags, std::string_view database, const Args &... args) const
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(flags, database.empty() ? params.current_database : database, args...);
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(context, flags, database.empty() ? params.current_database : database, args...);
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool ContextAccess::checkAccessImplHelper(const AccessRightsElement & element) const
|
||||
bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, const AccessRightsElement & element) const
|
||||
{
|
||||
assert(!element.grant_option || grant_option);
|
||||
if (element.isGlobalWithParameter())
|
||||
{
|
||||
if (element.any_parameter)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags);
|
||||
else
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags, element.parameter);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.parameter);
|
||||
}
|
||||
else if (element.any_database)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags);
|
||||
else if (element.any_table)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags, element.database);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.database);
|
||||
else if (element.any_column)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags, element.database, element.table);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.database, element.table);
|
||||
else
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags, element.database, element.table, element.columns);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.database, element.table, element.columns);
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool ContextAccess::checkAccessImpl(const AccessRightsElement & element) const
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessRightsElement & element) const
|
||||
{
|
||||
if constexpr (grant_option)
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, true>(element);
|
||||
return checkAccessImplHelper<throw_if_denied, true>(context, element);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (element.grant_option)
|
||||
return checkAccessImplHelper<throw_if_denied, true>(element);
|
||||
return checkAccessImplHelper<throw_if_denied, true>(context, element);
|
||||
else
|
||||
return checkAccessImplHelper<throw_if_denied, false>(element);
|
||||
return checkAccessImplHelper<throw_if_denied, false>(context, element);
|
||||
}
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessRightsElements & elements) const
|
||||
{
|
||||
for (const auto & element : elements)
|
||||
if (!checkAccessImpl<throw_if_denied, grant_option>(element))
|
||||
if (!checkAccessImpl<throw_if_denied, grant_option>(context, element))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl<false, false>(flags); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false>(flags, database); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false>(flags, database, table); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessRightsElement & element) const { return checkAccessImpl<false, false>(element); }
|
||||
bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return checkAccessImpl<false, false>(elements); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags) const { return checkAccessImpl<false, false>(context, flags); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false>(context, flags, database); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false>(context, flags, database, table); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false>(context, flags, database, table, column); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessRightsElement & element) const { return checkAccessImpl<false, false>(context, element); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessRightsElements & elements) const { return checkAccessImpl<false, false>(context, elements); }
|
||||
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags) const { return checkAccessImpl<false, true>(flags); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true>(flags, database); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true>(flags, database, table); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessRightsElement & element) const { return checkAccessImpl<false, true>(element); }
|
||||
bool ContextAccess::hasGrantOption(const AccessRightsElements & elements) const { return checkAccessImpl<false, true>(elements); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags) const { return checkAccessImpl<false, true>(context, flags); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true>(context, flags, database); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true>(context, flags, database, table); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true>(context, flags, database, table, column); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessRightsElement & element) const { return checkAccessImpl<false, true>(context, element); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const { return checkAccessImpl<false, true>(context, elements); }
|
||||
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl<true, false>(flags); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false>(flags, database); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false>(flags, database, table); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false>(flags, database, table, column); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl<true, false>(element); }
|
||||
void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl<true, false>(elements); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags) const { checkAccessImpl<true, false>(context, flags); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false>(context, flags, database); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false>(context, flags, database, table); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false>(context, flags, database, table, column); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessRightsElement & element) const { checkAccessImpl<true, false>(context, element); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessRightsElements & elements) const { checkAccessImpl<true, false>(context, elements); }
|
||||
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl<true, true>(flags); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true>(flags, database); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true>(flags, database, table); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true>(flags, database, table, column); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl<true, true>(element); }
|
||||
void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl<true, true>(elements); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags) const { checkAccessImpl<true, true>(context, flags); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true>(context, flags, database); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true>(context, flags, database, table); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true>(context, flags, database, table, column); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessRightsElement & element) const { checkAccessImpl<true, true>(context, element); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const { checkAccessImpl<true, true>(context, elements); }
|
||||
|
||||
|
||||
template <bool throw_if_denied, typename Container, typename GetNameFunction>
|
||||
bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const GetNameFunction & get_name_function) const
|
||||
bool ContextAccess::checkAdminOptionImplHelper(const ContextPtr & context, const Container & role_ids, const GetNameFunction & get_name_function) const
|
||||
{
|
||||
auto show_error = []<typename... FmtArgs>(int error_code [[maybe_unused]],
|
||||
FormatStringHelper<FmtArgs...> fmt_string [[maybe_unused]],
|
||||
@ -804,7 +810,7 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const
|
||||
if (!std::size(role_ids))
|
||||
return true;
|
||||
|
||||
if (isGranted(AccessType::ROLE_ADMIN))
|
||||
if (isGranted(context, AccessType::ROLE_ADMIN))
|
||||
return true;
|
||||
|
||||
auto info = getRolesInfo();
|
||||
@ -840,54 +846,54 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const UUID & role_id) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(to_array(role_id), [this](const UUID & id, size_t) { return access_control->tryReadName(id); });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, to_array(role_id), [this](const UUID & id, size_t) { return access_control->tryReadName(id); });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const UUID & role_id, const String & role_name) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id, const String & role_name) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(to_array(role_id), [&role_name](const UUID &, size_t) { return std::optional<String>{role_name}; });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, to_array(role_id), [&role_name](const UUID &, size_t) { return std::optional<String>{role_name}; });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(to_array(role_id), [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, to_array(role_id), [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const std::vector<UUID> & role_ids) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(role_ids, [this](const UUID & id, size_t) { return access_control->tryReadName(id); });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, role_ids, [this](const UUID & id, size_t) { return access_control->tryReadName(id); });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(role_ids, [&names_of_roles](const UUID &, size_t i) { return std::optional<String>{names_of_roles[i]}; });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, role_ids, [&names_of_roles](const UUID &, size_t i) { return std::optional<String>{names_of_roles[i]}; });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(role_ids, [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, role_ids, [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
}
|
||||
|
||||
bool ContextAccess::hasAdminOption(const UUID & role_id) const { return checkAdminOptionImpl<false>(role_id); }
|
||||
bool ContextAccess::hasAdminOption(const UUID & role_id, const String & role_name) const { return checkAdminOptionImpl<false>(role_id, role_name); }
|
||||
bool ContextAccess::hasAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { return checkAdminOptionImpl<false>(role_id, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const std::vector<UUID> & role_ids) const { return checkAdminOptionImpl<false>(role_ids); }
|
||||
bool ContextAccess::hasAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { return checkAdminOptionImpl<false>(role_ids, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { return checkAdminOptionImpl<false>(role_ids, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const UUID & role_id) const { return checkAdminOptionImpl<false>(context, role_id); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const UUID & role_id, const String & role_name) const { return checkAdminOptionImpl<false>(context, role_id, role_name); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { return checkAdminOptionImpl<false>(context, role_id, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids) const { return checkAdminOptionImpl<false>(context, role_ids); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { return checkAdminOptionImpl<false>(context, role_ids, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { return checkAdminOptionImpl<false>(context, role_ids, names_of_roles); }
|
||||
|
||||
void ContextAccess::checkAdminOption(const UUID & role_id) const { checkAdminOptionImpl<true>(role_id); }
|
||||
void ContextAccess::checkAdminOption(const UUID & role_id, const String & role_name) const { checkAdminOptionImpl<true>(role_id, role_name); }
|
||||
void ContextAccess::checkAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { checkAdminOptionImpl<true>(role_id, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids) const { checkAdminOptionImpl<true>(role_ids); }
|
||||
void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { checkAdminOptionImpl<true>(role_ids, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { checkAdminOptionImpl<true>(role_ids, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const UUID & role_id) const { checkAdminOptionImpl<true>(context, role_id); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const UUID & role_id, const String & role_name) const { checkAdminOptionImpl<true>(context, role_id, role_name); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { checkAdminOptionImpl<true>(context, role_id, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids) const { checkAdminOptionImpl<true>(context, role_ids); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { checkAdminOptionImpl<true>(context, role_ids, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { checkAdminOptionImpl<true>(context, role_ids, names_of_roles); }
|
||||
|
||||
|
||||
void ContextAccess::checkGranteeIsAllowed(const UUID & grantee_id, const IAccessEntity & grantee) const
|
||||
@ -919,4 +925,10 @@ void ContextAccess::checkGranteesAreAllowed(const std::vector<UUID> & grantee_id
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<const ContextAccessWrapper> ContextAccessWrapper::fromContext(const ContextPtr & context)
|
||||
{
|
||||
return context->getAccess();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -4,9 +4,12 @@
|
||||
#include <Access/ContextAccessParams.h>
|
||||
#include <Access/EnabledRowPolicies.h>
|
||||
#include <Interpreters/ClientInfo.h>
|
||||
#include <Access/QuotaUsage.h>
|
||||
#include <Common/SettingsChanges.h>
|
||||
#include <Core/UUID.h>
|
||||
#include <base/scope_guard.h>
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
@ -71,59 +74,59 @@ public:
|
||||
|
||||
/// Checks if a specified access is granted, and throws an exception if not.
|
||||
/// Empty database means the current database.
|
||||
void checkAccess(const AccessFlags & flags) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkAccess(const AccessRightsElement & element) const;
|
||||
void checkAccess(const AccessRightsElements & elements) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
void checkGrantOption(const AccessFlags & flags) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkGrantOption(const AccessRightsElement & element) const;
|
||||
void checkGrantOption(const AccessRightsElements & elements) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
/// Checks if a specified access is granted, and returns false if not.
|
||||
/// Empty database means the current database.
|
||||
bool isGranted(const AccessFlags & flags) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool isGranted(const AccessRightsElement & element) const;
|
||||
bool isGranted(const AccessRightsElements & elements) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
bool hasGrantOption(const AccessFlags & flags) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool hasGrantOption(const AccessRightsElement & element) const;
|
||||
bool hasGrantOption(const AccessRightsElements & elements) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
/// Checks if a specified role is granted with admin option, and throws an exception if not.
|
||||
void checkAdminOption(const UUID & role_id) const;
|
||||
void checkAdminOption(const UUID & role_id, const String & role_name) const;
|
||||
void checkAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
void checkAdminOption(const std::vector<UUID> & role_ids) const;
|
||||
void checkAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
void checkAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
void checkAdminOption(const ContextPtr & context, const UUID & role_id) const;
|
||||
void checkAdminOption(const ContextPtr & context, const UUID & role_id, const String & role_name) const;
|
||||
void checkAdminOption(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
void checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids) const;
|
||||
void checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
void checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
|
||||
/// Checks if a specified role is granted with admin option, and returns false if not.
|
||||
bool hasAdminOption(const UUID & role_id) const;
|
||||
bool hasAdminOption(const UUID & role_id, const String & role_name) const;
|
||||
bool hasAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool hasAdminOption(const std::vector<UUID> & role_ids) const;
|
||||
bool hasAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
bool hasAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const UUID & role_id) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const UUID & role_id, const String & role_name) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
|
||||
/// Checks if a grantee is allowed for the current user, throws an exception if not.
|
||||
void checkGranteeIsAllowed(const UUID & grantee_id, const IAccessEntity & grantee) const;
|
||||
@ -142,43 +145,43 @@ private:
|
||||
void calculateAccessRights() const TSA_REQUIRES(mutex);
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImpl(const AccessFlags & flags) const;
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const;
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessFlags & flags, std::string_view database, const Args &... args) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImpl(const AccessRightsElement & element) const;
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImpl(const AccessRightsElements & elements) const;
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool checkAccessImplHelper(AccessFlags flags, const Args &... args) const;
|
||||
bool checkAccessImplHelper(const ContextPtr & context, AccessFlags flags, const Args &... args) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImplHelper(const AccessRightsElement & element) const;
|
||||
bool checkAccessImplHelper(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const UUID & role_id) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const UUID & role_id, const String & role_name) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id, const String & role_name) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const std::vector<UUID> & role_ids) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
|
||||
template <bool throw_if_denied, typename Container, typename GetNameFunction>
|
||||
bool checkAdminOptionImplHelper(const Container & role_ids, const GetNameFunction & get_name_function) const;
|
||||
bool checkAdminOptionImplHelper(const ContextPtr & context, const Container & role_ids, const GetNameFunction & get_name_function) const;
|
||||
|
||||
const AccessControl * access_control = nullptr;
|
||||
const Params params;
|
||||
@ -203,4 +206,115 @@ private:
|
||||
mutable std::shared_ptr<const EnabledSettings> enabled_settings TSA_GUARDED_BY(mutex);
|
||||
};
|
||||
|
||||
/// This wrapper was added to be able to pass the current context to the access
|
||||
/// without the need to change the signature and all calls to the ContextAccess itself.
|
||||
/// Right now a context is used to store privileges that are checked for a query,
|
||||
/// and might be useful for something else in the future as well.
|
||||
class ContextAccessWrapper : public std::enable_shared_from_this<ContextAccessWrapper>
|
||||
{
|
||||
public:
|
||||
using ContextAccessPtr = std::shared_ptr<const ContextAccess>;
|
||||
|
||||
ContextAccessWrapper(const ContextAccessPtr & access_, const ContextPtr & context_): access(access_), context(context_) {}
|
||||
~ContextAccessWrapper() = default;
|
||||
|
||||
static std::shared_ptr<const ContextAccessWrapper> fromContext(const ContextPtr & context);
|
||||
|
||||
const ContextAccess::Params & getParams() const { return access->getParams(); }
|
||||
|
||||
const ContextAccessPtr & getAccess() const { return access; }
|
||||
|
||||
/// Returns the current user. Throws if user is nullptr.
|
||||
ALWAYS_INLINE UserPtr getUser() const { return access->getUser(); }
|
||||
/// Same as above, but can return nullptr.
|
||||
ALWAYS_INLINE UserPtr tryGetUser() const { return access->tryGetUser(); }
|
||||
ALWAYS_INLINE String getUserName() const { return access->getUserName(); }
|
||||
ALWAYS_INLINE std::optional<UUID> getUserID() const { return access->getUserID(); }
|
||||
|
||||
/// Returns information about current and enabled roles.
|
||||
ALWAYS_INLINE std::shared_ptr<const EnabledRolesInfo> getRolesInfo() const { return access->getRolesInfo(); }
|
||||
|
||||
/// Returns the row policy filter for a specified table.
|
||||
/// The function returns nullptr if there is no filter to apply.
|
||||
ALWAYS_INLINE RowPolicyFilterPtr getRowPolicyFilter(const String & database, const String & table_name, RowPolicyFilterType filter_type) const { return access->getRowPolicyFilter(database, table_name, filter_type); }
|
||||
|
||||
/// Returns the quota to track resource consumption.
|
||||
ALWAYS_INLINE std::shared_ptr<const EnabledQuota> getQuota() const { return access->getQuota(); }
|
||||
ALWAYS_INLINE std::optional<QuotaUsage> getQuotaUsage() const { return access->getQuotaUsage(); }
|
||||
|
||||
/// Returns the default settings, i.e. the settings which should be applied on user's login.
|
||||
ALWAYS_INLINE SettingsChanges getDefaultSettings() const { return access->getDefaultSettings(); }
|
||||
ALWAYS_INLINE std::shared_ptr<const SettingsProfilesInfo> getDefaultProfileInfo() const { return access->getDefaultProfileInfo(); }
|
||||
|
||||
/// Returns the current access rights.
|
||||
ALWAYS_INLINE std::shared_ptr<const AccessRights> getAccessRights() const { return access->getAccessRights(); }
|
||||
ALWAYS_INLINE std::shared_ptr<const AccessRights> getAccessRightsWithImplicit() const { return access->getAccessRightsWithImplicit(); }
|
||||
|
||||
/// Checks if a specified access is granted, and throws an exception if not.
|
||||
/// Empty database means the current database.
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags) const { access->checkAccess(context, flags); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database) const { access->checkAccess(context, flags, database); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { access->checkAccess(context, flags, database, table); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { access->checkAccess(context, flags, database, table, column); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { access->checkAccess(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { access->checkAccess(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessRightsElement & element) const { access->checkAccess(context, element); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessRightsElements & elements) const { access->checkAccess(context, elements); }
|
||||
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags) const { access->checkGrantOption(context, flags); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database) const { access->checkGrantOption(context, flags, database); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { access->checkGrantOption(context, flags, database, table); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { access->checkGrantOption(context, flags, database, table, column); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { access->checkGrantOption(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { access->checkGrantOption(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessRightsElement & element) const { access->checkGrantOption(context, element); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessRightsElements & elements) const { access->checkGrantOption(context, elements); }
|
||||
|
||||
/// Checks if a specified access is granted, and returns false if not.
|
||||
/// Empty database means the current database.
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags) const { return access->isGranted(context, flags); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database) const { return access->isGranted(context, flags, database); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return access->isGranted(context, flags, database, table); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return access->isGranted(context, flags, database, table, column); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return access->isGranted(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return access->isGranted(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessRightsElement & element) const { return access->isGranted(context, element); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessRightsElements & elements) const { return access->isGranted(context, elements); }
|
||||
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags) const { return access->hasGrantOption(context, flags); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database) const { return access->hasGrantOption(context, flags, database); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return access->hasGrantOption(context, flags, database, table); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return access->hasGrantOption(context, flags, database, table, column); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return access->hasGrantOption(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return access->hasGrantOption(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessRightsElement & element) const { return access->hasGrantOption(context, element); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessRightsElements & elements) const { return access->hasGrantOption(context, elements); }
|
||||
|
||||
/// Checks if a specified role is granted with admin option, and throws an exception if not.
|
||||
ALWAYS_INLINE void checkAdminOption(const UUID & role_id) const { access->checkAdminOption(context, role_id); }
|
||||
ALWAYS_INLINE void checkAdminOption(const UUID & role_id, const String & role_name) const { access->checkAdminOption(context, role_id, role_name); }
|
||||
ALWAYS_INLINE void checkAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { access->checkAdminOption(context, role_id, names_of_roles); }
|
||||
ALWAYS_INLINE void checkAdminOption(const std::vector<UUID> & role_ids) const { access->checkAdminOption(context, role_ids); }
|
||||
ALWAYS_INLINE void checkAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { access->checkAdminOption(context, role_ids, names_of_roles); }
|
||||
ALWAYS_INLINE void checkAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { access->checkAdminOption(context, role_ids, names_of_roles); }
|
||||
|
||||
/// Checks if a specified role is granted with admin option, and returns false if not.
|
||||
ALWAYS_INLINE bool hasAdminOption(const UUID & role_id) const { return access->hasAdminOption(context, role_id); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const UUID & role_id, const String & role_name) const { return access->hasAdminOption(context, role_id, role_name); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { return access->hasAdminOption(context, role_id, names_of_roles); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const std::vector<UUID> & role_ids) const { return access->hasAdminOption(context, role_ids); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { return access->hasAdminOption(context, role_ids, names_of_roles); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { return access->hasAdminOption(context, role_ids, names_of_roles); }
|
||||
|
||||
/// Checks if a grantee is allowed for the current user, throws an exception if not.
|
||||
ALWAYS_INLINE void checkGranteeIsAllowed(const UUID & grantee_id, const IAccessEntity & grantee) const { access->checkGranteeIsAllowed(grantee_id, grantee); }
|
||||
/// Checks if grantees are allowed for the current user, throws an exception if not.
|
||||
ALWAYS_INLINE void checkGranteesAreAllowed(const std::vector<UUID> & grantee_ids) const { access->checkGranteesAreAllowed(grantee_ids); }
|
||||
|
||||
private:
|
||||
ContextAccessPtr access;
|
||||
ContextPtr context;
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -15,22 +15,8 @@ namespace ErrorCodes
|
||||
|
||||
bool operator==(const SettingsProfilesInfo & lhs, const SettingsProfilesInfo & rhs)
|
||||
{
|
||||
if (lhs.settings != rhs.settings)
|
||||
return false;
|
||||
|
||||
if (lhs.constraints != rhs.constraints)
|
||||
return false;
|
||||
|
||||
if (lhs.profiles != rhs.profiles)
|
||||
return false;
|
||||
|
||||
if (lhs.profiles_with_implicit != rhs.profiles_with_implicit)
|
||||
return false;
|
||||
|
||||
if (lhs.names_of_profiles != rhs.names_of_profiles)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
return std::tie(lhs.settings, lhs.constraints, lhs.profiles, lhs.profiles_with_implicit, lhs.names_of_profiles)
|
||||
== std::tie(rhs.settings, rhs.constraints, rhs.profiles, rhs.profiles_with_implicit, rhs.names_of_profiles);
|
||||
}
|
||||
|
||||
std::shared_ptr<const SettingsConstraintsAndProfileIDs>
|
||||
@ -66,18 +52,20 @@ Strings SettingsProfilesInfo::getProfileNames() const
|
||||
{
|
||||
Strings result;
|
||||
result.reserve(profiles.size());
|
||||
for (const auto & profile_id : profiles)
|
||||
for (const UUID & profile_uuid : profiles)
|
||||
{
|
||||
const auto p = names_of_profiles.find(profile_id);
|
||||
if (p != names_of_profiles.end())
|
||||
result.push_back(p->second);
|
||||
const auto names_it = names_of_profiles.find(profile_uuid);
|
||||
if (names_it != names_of_profiles.end())
|
||||
{
|
||||
result.push_back(names_it->second);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (const auto name = access_control.tryReadName(profile_id))
|
||||
if (const auto name = access_control.tryReadName(profile_uuid))
|
||||
// We could've updated cache here, but it is a very rare case, so don't bother.
|
||||
result.push_back(*name);
|
||||
else
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unable to get profile name for {}", toString(profile_id));
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unable to get profile name for {}", toString(profile_uuid));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -29,7 +29,11 @@ struct SettingsProfilesInfo
|
||||
/// Names of all the profiles in `profiles`.
|
||||
std::unordered_map<UUID, String> names_of_profiles;
|
||||
|
||||
explicit SettingsProfilesInfo(const AccessControl & access_control_) : constraints(access_control_), access_control(access_control_) {}
|
||||
explicit SettingsProfilesInfo(const AccessControl & access_control_)
|
||||
: constraints(access_control_), access_control(access_control_)
|
||||
{
|
||||
}
|
||||
|
||||
std::shared_ptr<const SettingsConstraintsAndProfileIDs> getConstraintsAndProfileIDs(
|
||||
const std::shared_ptr<const SettingsConstraintsAndProfileIDs> & previous = nullptr) const;
|
||||
|
||||
|
||||
@ -1,265 +0,0 @@
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/FactoryHelpers.h>
|
||||
|
||||
#include <Columns/IColumn.h>
|
||||
#include <Columns/ColumnNullable.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
|
||||
#include <Core/ServerSettings.h>
|
||||
#include <Core/ColumnWithTypeAndName.h>
|
||||
|
||||
#include <Common/ArenaAllocator.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Interpreters/castColumn.h>
|
||||
|
||||
#include <DataTypes/IDataType.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
struct GroupConcatDataBase
|
||||
{
|
||||
UInt64 data_size = 0;
|
||||
UInt64 allocated_size = 0;
|
||||
char * data = nullptr;
|
||||
|
||||
void checkAndUpdateSize(UInt64 add, Arena * arena)
|
||||
{
|
||||
if (data_size + add >= allocated_size)
|
||||
{
|
||||
auto old_size = allocated_size;
|
||||
allocated_size = std::max(2 * allocated_size, data_size + add);
|
||||
data = arena->realloc(data, old_size, allocated_size);
|
||||
}
|
||||
}
|
||||
|
||||
void insertChar(const char * str, UInt64 str_size, Arena * arena)
|
||||
{
|
||||
checkAndUpdateSize(str_size, arena);
|
||||
memcpy(data + data_size, str, str_size);
|
||||
data_size += str_size;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
struct GroupConcatData : public GroupConcatDataBase
|
||||
{
|
||||
using Offset = UInt64;
|
||||
using Allocator = MixedAlignedArenaAllocator<alignof(Offset), 4096>;
|
||||
using Offsets = PODArray<Offset, 32, Allocator>;
|
||||
|
||||
/// offset[i * 2] - beginning of the i-th row, offset[i * 2 + 1] - end of the i-th row
|
||||
Offsets offsets;
|
||||
UInt64 num_rows = 0;
|
||||
|
||||
UInt64 getSize(size_t i) const { return offsets[i * 2 + 1] - offsets[i * 2]; }
|
||||
|
||||
UInt64 getString(size_t i) const { return offsets[i * 2]; }
|
||||
|
||||
void insert(const IColumn * column, const SerializationPtr & serialization, size_t row_num, Arena * arena)
|
||||
{
|
||||
WriteBufferFromOwnString buff;
|
||||
serialization->serializeText(*column, row_num, buff, {});
|
||||
auto string = buff.stringView();
|
||||
|
||||
checkAndUpdateSize(string.size(), arena);
|
||||
memcpy(data + data_size, string.data(), string.size());
|
||||
offsets.push_back(data_size, arena);
|
||||
data_size += string.size();
|
||||
offsets.push_back(data_size, arena);
|
||||
num_rows++;
|
||||
}
|
||||
};
|
||||
|
||||
template <bool has_limit>
|
||||
class GroupConcatImpl final
|
||||
: public IAggregateFunctionDataHelper<GroupConcatData, GroupConcatImpl<has_limit>>
|
||||
{
|
||||
static constexpr auto name = "groupConcat";
|
||||
|
||||
SerializationPtr serialization;
|
||||
UInt64 limit;
|
||||
const String delimiter;
|
||||
|
||||
public:
|
||||
GroupConcatImpl(const DataTypePtr & data_type_, const Array & parameters_, UInt64 limit_, const String & delimiter_)
|
||||
: IAggregateFunctionDataHelper<GroupConcatData, GroupConcatImpl<has_limit>>(
|
||||
{data_type_}, parameters_, std::make_shared<DataTypeString>())
|
||||
, serialization(this->argument_types[0]->getDefaultSerialization())
|
||||
, limit(limit_)
|
||||
, delimiter(delimiter_)
|
||||
{
|
||||
}
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override
|
||||
{
|
||||
auto & cur_data = this->data(place);
|
||||
|
||||
if constexpr (has_limit)
|
||||
if (cur_data.num_rows >= limit)
|
||||
return;
|
||||
|
||||
if (cur_data.data_size != 0)
|
||||
cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena);
|
||||
|
||||
cur_data.insert(columns[0], serialization, row_num, arena);
|
||||
}
|
||||
|
||||
void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override
|
||||
{
|
||||
auto & cur_data = this->data(place);
|
||||
auto & rhs_data = this->data(rhs);
|
||||
|
||||
if (rhs_data.data_size == 0)
|
||||
return;
|
||||
|
||||
if constexpr (has_limit)
|
||||
{
|
||||
UInt64 new_elems_count = std::min(rhs_data.num_rows, limit - cur_data.num_rows);
|
||||
for (UInt64 i = 0; i < new_elems_count; ++i)
|
||||
{
|
||||
if (cur_data.data_size != 0)
|
||||
cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena);
|
||||
|
||||
cur_data.offsets.push_back(cur_data.data_size, arena);
|
||||
cur_data.insertChar(rhs_data.data + rhs_data.getString(i), rhs_data.getSize(i), arena);
|
||||
cur_data.num_rows++;
|
||||
cur_data.offsets.push_back(cur_data.data_size, arena);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (cur_data.data_size != 0)
|
||||
cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena);
|
||||
|
||||
cur_data.insertChar(rhs_data.data, rhs_data.data_size, arena);
|
||||
}
|
||||
}
|
||||
|
||||
void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional<size_t> /* version */) const override
|
||||
{
|
||||
auto & cur_data = this->data(place);
|
||||
|
||||
writeVarUInt(cur_data.data_size, buf);
|
||||
writeVarUInt(cur_data.allocated_size, buf);
|
||||
|
||||
buf.write(cur_data.data, cur_data.data_size);
|
||||
|
||||
if constexpr (has_limit)
|
||||
{
|
||||
writeVarUInt(cur_data.num_rows, buf);
|
||||
for (const auto & offset : cur_data.offsets)
|
||||
writeVarUInt(offset, buf);
|
||||
}
|
||||
}
|
||||
|
||||
void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional<size_t> /* version */, Arena * arena) const override
|
||||
{
|
||||
auto & cur_data = this->data(place);
|
||||
|
||||
readVarUInt(cur_data.data_size, buf);
|
||||
readVarUInt(cur_data.allocated_size, buf);
|
||||
|
||||
buf.readStrict(cur_data.data, cur_data.data_size);
|
||||
|
||||
if constexpr (has_limit)
|
||||
{
|
||||
readVarUInt(cur_data.num_rows, buf);
|
||||
cur_data.offsets.resize_exact(cur_data.num_rows * 2, arena);
|
||||
for (auto & offset : cur_data.offsets)
|
||||
readVarUInt(offset, buf);
|
||||
}
|
||||
}
|
||||
|
||||
void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
|
||||
{
|
||||
auto & cur_data = this->data(place);
|
||||
|
||||
if (cur_data.data_size == 0)
|
||||
{
|
||||
auto column_nullable = IColumn::mutate(makeNullable(to.getPtr()));
|
||||
column_nullable->insertDefault();
|
||||
return;
|
||||
}
|
||||
|
||||
auto & column_string = assert_cast<ColumnString &>(to);
|
||||
column_string.insertData(cur_data.data, cur_data.data_size);
|
||||
}
|
||||
|
||||
bool allocatesMemoryInArena() const override { return true; }
|
||||
};
|
||||
|
||||
AggregateFunctionPtr createAggregateFunctionGroupConcat(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *)
|
||||
{
|
||||
assertUnary(name, argument_types);
|
||||
|
||||
bool has_limit = false;
|
||||
UInt64 limit = 0;
|
||||
String delimiter;
|
||||
|
||||
if (parameters.size() > 2)
|
||||
throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION,
|
||||
"Incorrect number of parameters for aggregate function {}, should be 0, 1 or 2, got: {}", name, parameters.size());
|
||||
|
||||
if (!parameters.empty())
|
||||
{
|
||||
auto type = parameters[0].getType();
|
||||
if (type != Field::Types::String)
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First parameter for aggregate function {} should be string", name);
|
||||
|
||||
delimiter = parameters[0].get<String>();
|
||||
}
|
||||
if (parameters.size() == 2)
|
||||
{
|
||||
auto type = parameters[1].getType();
|
||||
|
||||
if (type != Field::Types::Int64 && type != Field::Types::UInt64)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second parameter for aggregate function {} should be a positive number", name);
|
||||
|
||||
if ((type == Field::Types::Int64 && parameters[1].get<Int64>() <= 0) ||
|
||||
(type == Field::Types::UInt64 && parameters[1].get<UInt64>() == 0))
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second parameter for aggregate function {} should be a positive number, got: {}", name, parameters[1].get<Int64>());
|
||||
|
||||
has_limit = true;
|
||||
limit = parameters[1].get<UInt64>();
|
||||
}
|
||||
|
||||
if (has_limit)
|
||||
return std::make_shared<GroupConcatImpl</* has_limit= */ true>>(argument_types[0], parameters, limit, delimiter);
|
||||
else
|
||||
return std::make_shared<GroupConcatImpl</* has_limit= */ false>>(argument_types[0], parameters, limit, delimiter);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void registerAggregateFunctionGroupConcat(AggregateFunctionFactory & factory)
|
||||
{
|
||||
AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
|
||||
|
||||
factory.registerFunction("groupConcat", { createAggregateFunctionGroupConcat, properties });
|
||||
factory.registerAlias("group_concat", "groupConcat", AggregateFunctionFactory::CaseInsensitive);
|
||||
}
|
||||
|
||||
}
|
||||
@ -228,6 +228,11 @@ public:
|
||||
return prefix_size + nested_func->sizeOfData();
|
||||
}
|
||||
|
||||
size_t alignOfData() const override
|
||||
{
|
||||
return std::max(alignof(Data), nested_func->alignOfData());
|
||||
}
|
||||
|
||||
void create(AggregateDataPtr __restrict place) const override
|
||||
{
|
||||
new (place) Data;
|
||||
|
||||
@ -19,7 +19,6 @@ void registerAggregateFunctionGroupArraySorted(AggregateFunctionFactory & factor
|
||||
void registerAggregateFunctionGroupUniqArray(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionGroupArrayInsertAt(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionGroupArrayIntersect(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionGroupConcat(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsQuantile(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsQuantileDeterministic(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsQuantileExact(AggregateFunctionFactory &);
|
||||
@ -121,7 +120,6 @@ void registerAggregateFunctions()
|
||||
registerAggregateFunctionGroupUniqArray(factory);
|
||||
registerAggregateFunctionGroupArrayInsertAt(factory);
|
||||
registerAggregateFunctionGroupArrayIntersect(factory);
|
||||
registerAggregateFunctionGroupConcat(factory);
|
||||
registerAggregateFunctionsQuantile(factory);
|
||||
registerAggregateFunctionsQuantileDeterministic(factory);
|
||||
registerAggregateFunctionsQuantileExact(factory);
|
||||
|
||||
@ -602,6 +602,8 @@
|
||||
M(721, DEPRECATED_FUNCTION) \
|
||||
M(722, ASYNC_LOAD_WAIT_FAILED) \
|
||||
M(723, PARQUET_EXCEPTION) \
|
||||
M(724, TOO_MANY_TABLES) \
|
||||
M(725, TOO_MANY_DATABASES) \
|
||||
\
|
||||
M(900, DISTRIBUTED_CACHE_ERROR) \
|
||||
M(901, CANNOT_USE_DISTRIBUTED_CACHE) \
|
||||
|
||||
161
src/Common/HilbertUtils.h
Normal file
161
src/Common/HilbertUtils.h
Normal file
@ -0,0 +1,161 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
#include <Common/BitHelpers.h>
|
||||
#include "base/types.h"
|
||||
#include <Functions/hilbertDecode2DLUT.h>
|
||||
#include <base/defines.h>
|
||||
#include <array>
|
||||
#include <set>
|
||||
|
||||
|
||||
namespace HilbertDetails
|
||||
{
|
||||
|
||||
struct Segment // represents [begin; end], all bounds are included
|
||||
{
|
||||
UInt64 begin;
|
||||
UInt64 end;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
Given the range of values of hilbert code - and this function will return segments of the Hilbert curve
|
||||
such that each of them lies in a whole domain (aka square)
|
||||
0 1
|
||||
┌────────────────────────────────┐
|
||||
│ │ │
|
||||
│ │ │
|
||||
0 │ 00xxx │ 11xxx │
|
||||
│ | │ | │
|
||||
│ | │ | │
|
||||
│_______________│________________│
|
||||
│ | │ | │
|
||||
│ | │ | │
|
||||
│ | │ | │
|
||||
1 │ 01xxx______│_____10xxx │
|
||||
│ │ │
|
||||
│ │ │
|
||||
└────────────────────────────────┘
|
||||
Imagine a square, one side of which is a x-axis, other is a y-axis.
|
||||
First approximation of the Hilbert curve is on the picture - U curve.
|
||||
So we divide Hilbert Code Interval on 4 parts each of which is represented by a square
|
||||
and look where the given interval [start, finish] is located:
|
||||
[00xxxxxx | 01xxxxxx | 10xxxxxx | 11xxxxxx ]
|
||||
1: [ ]
|
||||
start = 0010111 end = 10111110
|
||||
2: [ ] [ ]
|
||||
If it contains a whole sector (that represents a domain=square),
|
||||
then we take this range. In the example above - it is a sector [01000000, 01111111]
|
||||
Then we dig into the recursion and check the remaining ranges.
|
||||
Note that after the first call all other ranges in the recursion will have either start or finish on the end of a range,
|
||||
so the complexity of the algorithm will be O(logN), where N is the maximum of hilbert code.
|
||||
*/
|
||||
template <typename F>
|
||||
void segmentBinaryPartition(UInt64 start, UInt64 finish, UInt8 current_bits, F && callback)
|
||||
{
|
||||
if (current_bits == 0)
|
||||
return;
|
||||
|
||||
const auto next_bits = current_bits - 2;
|
||||
const auto history = current_bits == 64 ? 0 : (start >> current_bits) << current_bits;
|
||||
|
||||
const auto chunk_mask = 0b11;
|
||||
const auto start_chunk = (start >> next_bits) & chunk_mask;
|
||||
const auto finish_chunk = (finish >> next_bits) & chunk_mask;
|
||||
|
||||
auto construct_range = [next_bits, history](UInt64 chunk)
|
||||
{
|
||||
return HilbertDetails::Segment{
|
||||
.begin = history + (chunk << next_bits),
|
||||
.end = history + ((chunk + 1) << next_bits) - 1
|
||||
};
|
||||
};
|
||||
|
||||
if (start_chunk == finish_chunk)
|
||||
{
|
||||
if ((finish - start + 1) == (1 << next_bits)) // it means that [begin, end] is a range
|
||||
{
|
||||
callback(HilbertDetails::Segment{.begin = start, .end = finish});
|
||||
return;
|
||||
}
|
||||
segmentBinaryPartition(start, finish, next_bits, callback);
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto range_chunk = start_chunk + 1; range_chunk < finish_chunk; ++range_chunk)
|
||||
{
|
||||
callback(construct_range(range_chunk));
|
||||
}
|
||||
|
||||
const auto start_range = construct_range(start_chunk);
|
||||
if (start == start_range.begin)
|
||||
{
|
||||
callback(start_range);
|
||||
}
|
||||
else
|
||||
{
|
||||
segmentBinaryPartition(start, start_range.end, next_bits, callback);
|
||||
}
|
||||
|
||||
const auto finish_range = construct_range(finish_chunk);
|
||||
if (finish == finish_range.end)
|
||||
{
|
||||
callback(finish_range);
|
||||
}
|
||||
else
|
||||
{
|
||||
segmentBinaryPartition(finish_range.begin, finish, next_bits, callback);
|
||||
}
|
||||
}
|
||||
|
||||
// Given 2 points representing ends of the range of Hilbert Curve that lies in a whole domain.
|
||||
// The are neighbour corners of some square - and the function returns ranges of both sides of this square
|
||||
inline std::array<std::pair<UInt64, UInt64>, 2> createRangeFromCorners(UInt64 x1, UInt64 y1, UInt64 x2, UInt64 y2)
|
||||
{
|
||||
UInt64 dist_x = x1 > x2 ? x1 - x2 : x2 - x1;
|
||||
UInt64 dist_y = y1 > y2 ? y1 - y2 : y2 - y1;
|
||||
UInt64 range_size = std::max(dist_x, dist_y);
|
||||
bool contains_minimum_vertice = x1 % (range_size + 1) == 0;
|
||||
if (contains_minimum_vertice)
|
||||
{
|
||||
UInt64 x_min = std::min(x1, x2);
|
||||
UInt64 y_min = std::min(y1, y2);
|
||||
return {
|
||||
std::pair<UInt64, UInt64>{x_min, x_min + range_size},
|
||||
std::pair<UInt64, UInt64>{y_min, y_min + range_size}
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
UInt64 x_max = std::max(x1, x2);
|
||||
UInt64 y_max = std::max(y1, y2);
|
||||
chassert(x_max >= range_size);
|
||||
chassert(y_max >= range_size);
|
||||
return {
|
||||
std::pair<UInt64, UInt64>{x_max - range_size, x_max},
|
||||
std::pair<UInt64, UInt64>{y_max - range_size, y_max}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Unpack an interval of Hilbert curve to hyperrectangles covered by it across N dimensions.
|
||||
*/
|
||||
template <typename F>
|
||||
void hilbertIntervalToHyperrectangles2D(UInt64 first, UInt64 last, F && callback)
|
||||
{
|
||||
const auto equal_bits_count = getLeadingZeroBits(last | first);
|
||||
const auto even_equal_bits_count = equal_bits_count - equal_bits_count % 2;
|
||||
segmentBinaryPartition(first, last, 64 - even_equal_bits_count, [&](HilbertDetails::Segment range)
|
||||
{
|
||||
auto interval1 = DB::FunctionHilbertDecode2DWIthLookupTableImpl<3>::decode(range.begin);
|
||||
auto interval2 = DB::FunctionHilbertDecode2DWIthLookupTableImpl<3>::decode(range.end);
|
||||
|
||||
std::array<std::pair<UInt64, UInt64>, 2> unpacked_range = createRangeFromCorners(
|
||||
std::get<0>(interval1), std::get<1>(interval1),
|
||||
std::get<0>(interval2), std::get<1>(interval2));
|
||||
|
||||
callback(unpacked_range);
|
||||
});
|
||||
}
|
||||
@ -406,13 +406,19 @@ private:
|
||||
void processQueue(std::unique_lock<std::mutex> && lock)
|
||||
{
|
||||
if (events.empty())
|
||||
return processActivation(std::move(lock));
|
||||
{
|
||||
processActivation(std::move(lock));
|
||||
return;
|
||||
}
|
||||
if (activations.empty())
|
||||
return processEvent(std::move(lock));
|
||||
{
|
||||
processEvent(std::move(lock));
|
||||
return;
|
||||
}
|
||||
if (activations.front().activation_event_id < events.front().event_id)
|
||||
return processActivation(std::move(lock));
|
||||
processActivation(std::move(lock));
|
||||
else
|
||||
return processEvent(std::move(lock));
|
||||
processEvent(std::move(lock));
|
||||
}
|
||||
|
||||
void processActivation(std::unique_lock<std::mutex> && lock)
|
||||
|
||||
@ -102,6 +102,8 @@ namespace DB
|
||||
M(UInt64, max_dictionary_num_to_warn, 1000lu, "If the number of dictionaries is greater than this value, the server will create a warning that will displayed to user.", 0) \
|
||||
M(UInt64, max_database_num_to_warn, 1000lu, "If the number of databases is greater than this value, the server will create a warning that will displayed to user.", 0) \
|
||||
M(UInt64, max_part_num_to_warn, 100000lu, "If the number of parts is greater than this value, the server will create a warning that will displayed to user.", 0) \
|
||||
M(UInt64, max_table_num_to_throw, 0lu, "If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \
|
||||
M(UInt64, max_database_num_to_throw, 0lu, "If number of databases is greater than this value, server will throw an exception. 0 means no limitation.", 0) \
|
||||
M(UInt64, concurrent_threads_soft_limit_num, 0, "Sets how many concurrent thread can be allocated before applying CPU pressure. Zero means unlimited.", 0) \
|
||||
M(UInt64, concurrent_threads_soft_limit_ratio_to_cores, 0, "Same as concurrent_threads_soft_limit_num, but with ratio to cores.", 0) \
|
||||
\
|
||||
|
||||
@ -933,6 +933,7 @@ class IColumn;
|
||||
M(Int64, prefer_warmed_unmerged_parts_seconds, 0, "Only available in ClickHouse Cloud. If a merged part is less than this many seconds old and is not pre-warmed (see cache_populated_by_fetch), but all its source parts are available and pre-warmed, SELECT queries will read from those parts instead. Only for ReplicatedMergeTree. Note that this only checks whether CacheWarmer processed the part; if the part was fetched into cache by something else, it'll still be considered cold until CacheWarmer gets to it; if it was warmed, then evicted from cache, it'll still be considered warm.", 0) \
|
||||
M(Bool, iceberg_engine_ignore_schema_evolution, false, "Ignore schema evolution in Iceberg table engine and read all data using latest schema saved on table creation. Note that it can lead to incorrect result", 0) \
|
||||
M(Bool, allow_deprecated_error_prone_window_functions, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)", 0) \
|
||||
M(Bool, uniform_snowflake_conversion_functions, true, "Enables functions snowflakeIDToDateTime[64] and dateTime[64]ToSnowflakeID while disabling functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake.", 0) \
|
||||
|
||||
// End of COMMON_SETTINGS
|
||||
// Please add settings related to formats into the FORMAT_FACTORY_SETTINGS, move obsolete settings to OBSOLETE_SETTINGS and obsolete format settings to OBSOLETE_FORMAT_SETTINGS.
|
||||
|
||||
@ -101,6 +101,7 @@ static const std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges
|
||||
{"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."},
|
||||
{"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."},
|
||||
{"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"},
|
||||
{"uniform_snowflake_conversion_functions", false, true, "Enable functions snowflakeIDToDateTime[64] and dateTime[64]ToSnowflakeID."},
|
||||
{"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."},
|
||||
{"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."},
|
||||
{"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."},
|
||||
|
||||
@ -186,6 +186,7 @@ void DatabaseLazy::attachTable(ContextPtr /* context_ */, const String & table_n
|
||||
throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists.", backQuote(database_name), backQuote(table_name));
|
||||
|
||||
it->second.expiration_iterator = cache_expiration_queue.emplace(cache_expiration_queue.end(), current_time, table_name);
|
||||
|
||||
CurrentMetrics::add(CurrentMetrics::AttachedTable, 1);
|
||||
}
|
||||
|
||||
@ -202,6 +203,7 @@ StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & ta
|
||||
if (it->second.expiration_iterator != cache_expiration_queue.end())
|
||||
cache_expiration_queue.erase(it->second.expiration_iterator);
|
||||
tables_cache.erase(it);
|
||||
|
||||
CurrentMetrics::sub(CurrentMetrics::AttachedTable, 1);
|
||||
}
|
||||
return res;
|
||||
|
||||
@ -73,9 +73,10 @@ zkutil::ZooKeeperPtr DatabaseReplicated::getZooKeeper() const
|
||||
return getContext()->getZooKeeper();
|
||||
}
|
||||
|
||||
static inline String getHostID(ContextPtr global_context, const UUID & db_uuid)
|
||||
static inline String getHostID(ContextPtr global_context, const UUID & db_uuid, bool secure)
|
||||
{
|
||||
return Cluster::Address::toString(getFQDNOrHostName(), global_context->getTCPPort()) + ':' + toString(db_uuid);
|
||||
UInt16 port = secure ? global_context->getTCPPortSecure().value_or(DBMS_DEFAULT_SECURE_PORT) : global_context->getTCPPort();
|
||||
return Cluster::Address::toString(getFQDNOrHostName(), port) + ':' + toString(db_uuid);
|
||||
}
|
||||
|
||||
static inline UInt64 getMetadataHash(const String & table_name, const String & metadata)
|
||||
@ -415,8 +416,10 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessL
|
||||
return;
|
||||
}
|
||||
|
||||
String host_id = getHostID(getContext(), db_uuid);
|
||||
if (is_create_query || replica_host_id != host_id)
|
||||
String host_id = getHostID(getContext(), db_uuid, cluster_auth_info.cluster_secure_connection);
|
||||
String host_id_default = getHostID(getContext(), db_uuid, false);
|
||||
|
||||
if (is_create_query || (replica_host_id != host_id && replica_host_id != host_id_default))
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::REPLICA_ALREADY_EXISTS,
|
||||
@ -424,6 +427,14 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessL
|
||||
replica_name, shard_name, zookeeper_path, replica_host_id, host_id);
|
||||
}
|
||||
|
||||
/// Before 24.6 we always created host_id with insecure port, even if cluster_auth_info.cluster_secure_connection was true.
|
||||
/// So not to break compatibility, we need to update host_id to secure one if cluster_auth_info.cluster_secure_connection is true.
|
||||
if (host_id != host_id_default && replica_host_id == host_id_default)
|
||||
{
|
||||
current_zookeeper->set(replica_path, host_id, -1);
|
||||
createEmptyLogEntry(current_zookeeper);
|
||||
}
|
||||
|
||||
/// Check that replica_group_name in ZooKeeper matches the local one and change it if necessary.
|
||||
String zk_replica_group_name;
|
||||
if (!current_zookeeper->tryGet(replica_path + "/replica_group", zk_replica_group_name))
|
||||
@ -550,7 +561,7 @@ void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPt
|
||||
"already contains some data and it does not look like Replicated database path.", zookeeper_path);
|
||||
|
||||
/// Write host name to replica_path, it will protect from multiple replicas with the same name
|
||||
auto host_id = getHostID(getContext(), db_uuid);
|
||||
auto host_id = getHostID(getContext(), db_uuid, cluster_auth_info.cluster_secure_connection);
|
||||
|
||||
for (int attempts = 10; attempts > 0; --attempts)
|
||||
{
|
||||
|
||||
@ -260,7 +260,9 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
|
||||
res = it->second;
|
||||
tables.erase(it);
|
||||
res->is_detached = true;
|
||||
CurrentMetrics::sub(getAttachedCounterForStorage(res), 1);
|
||||
|
||||
if (res->isSystemStorage() == false)
|
||||
CurrentMetrics::sub(getAttachedCounterForStorage(res), 1);
|
||||
|
||||
auto table_id = res->getStorageID();
|
||||
if (table_id.hasUUID())
|
||||
@ -301,7 +303,9 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
|
||||
/// It is important to reset is_detached here since in case of RENAME in
|
||||
/// non-Atomic database the is_detached is set to true before RENAME.
|
||||
table->is_detached = false;
|
||||
CurrentMetrics::add(getAttachedCounterForStorage(table), 1);
|
||||
|
||||
if (table->isSystemStorage() == false && table_id.database_name != DatabaseCatalog::SYSTEM_DATABASE)
|
||||
CurrentMetrics::add(getAttachedCounterForStorage(table), 1);
|
||||
}
|
||||
|
||||
void DatabaseWithOwnTablesBase::shutdown()
|
||||
|
||||
181
src/Functions/dateTimeToSnowflakeID.cpp
Normal file
181
src/Functions/dateTimeToSnowflakeID.cpp
Normal file
@ -0,0 +1,181 @@
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Columns/ColumnsDateTime.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int UNKNOWN_FUNCTION;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/// See generateSnowflakeID.cpp
|
||||
constexpr int time_shift = 22;
|
||||
|
||||
}
|
||||
|
||||
class FunctionDateTimeToSnowflakeID : public IFunction
|
||||
{
|
||||
private:
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
static constexpr auto name = "dateTimeToSnowflakeID";
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionDateTimeToSnowflakeID>(context); }
|
||||
explicit FunctionDateTimeToSnowflakeID(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool isVariadic() const override { return true; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateTime), nullptr, "DateTime"}
|
||||
};
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"epoch", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), isColumnConst, "UInt*"}
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args, optional_args);
|
||||
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (!uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "To use function {}, setting 'uniform_snowflake_conversion_functions' must be enabled", getName());
|
||||
|
||||
const auto & col_src = *arguments[0].column;
|
||||
|
||||
size_t epoch = 0;
|
||||
if (arguments.size() == 2 && input_rows_count != 0)
|
||||
{
|
||||
const auto & col_epoch = *arguments[1].column;
|
||||
epoch = col_epoch.getUInt(0);
|
||||
}
|
||||
|
||||
auto col_res = ColumnUInt64::create(input_rows_count);
|
||||
auto & res_data = col_res->getData();
|
||||
|
||||
const auto & src_data = typeid_cast<const ColumnDateTime &>(col_src).getData();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = (static_cast<UInt64>(src_data[i]) * 1000 - epoch) << time_shift;
|
||||
return col_res;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class FunctionDateTime64ToSnowflakeID : public IFunction
|
||||
{
|
||||
private:
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
static constexpr auto name = "dateTime64ToSnowflakeID";
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionDateTime64ToSnowflakeID>(context); }
|
||||
explicit FunctionDateTime64ToSnowflakeID(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool isVariadic() const override { return true; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateTime64), nullptr, "DateTime64"}
|
||||
};
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"epoch", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), isColumnConst, "UInt*"}
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args, optional_args);
|
||||
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (!uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "To use function {}, setting 'uniform_snowflake_conversion_functions' must be enabled", getName());
|
||||
|
||||
const auto & col_src = *arguments[0].column;
|
||||
const auto & src_data = typeid_cast<const ColumnDateTime64 &>(col_src).getData();
|
||||
|
||||
size_t epoch = 0;
|
||||
if (arguments.size() == 2 && input_rows_count != 0)
|
||||
{
|
||||
const auto & col_epoch = *arguments[1].column;
|
||||
epoch = col_epoch.getUInt(0);
|
||||
}
|
||||
|
||||
auto col_res = ColumnUInt64::create(input_rows_count);
|
||||
auto & res_data = col_res->getData();
|
||||
|
||||
/// timestamps in snowflake-ids are millisecond-based, convert input to milliseconds
|
||||
UInt32 src_scale = getDecimalScale(*arguments[0].type);
|
||||
Int64 multiplier_msec = DecimalUtils::scaleMultiplier<DateTime64>(3);
|
||||
Int64 multiplier_src = DecimalUtils::scaleMultiplier<DateTime64>(src_scale);
|
||||
auto factor = multiplier_msec / static_cast<double>(multiplier_src);
|
||||
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = static_cast<UInt64>(src_data[i] * factor - epoch) << time_shift;
|
||||
|
||||
return col_res;
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_FUNCTION(DateTimeToSnowflakeID)
|
||||
{
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Converts a [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.)";
|
||||
FunctionDocumentation::Syntax syntax = "dateTimeToSnowflakeID(value[, epoch])";
|
||||
FunctionDocumentation::Arguments arguments = {
|
||||
{"value", "Date with time. [DateTime](../data-types/datetime.md)."},
|
||||
{"epoch", "Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md)"}
|
||||
};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.";
|
||||
FunctionDocumentation::Examples examples = {{"simple", "SELECT dateTimeToSnowflakeID(toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai'))", "6832626392367104000"}};
|
||||
FunctionDocumentation::Categories categories = {"Snowflake ID"};
|
||||
|
||||
factory.registerFunction<FunctionDateTimeToSnowflakeID>({description, syntax, arguments, returned_value, examples, categories});
|
||||
}
|
||||
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Converts a [DateTime64](../data-types/datetime64.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.)";
|
||||
FunctionDocumentation::Syntax syntax = "dateTime64ToSnowflakeID(value[, epoch])";
|
||||
FunctionDocumentation::Arguments arguments = {
|
||||
{"value", "Date with time. [DateTime64](../data-types/datetime.md)."},
|
||||
{"epoch", "Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md)"}
|
||||
};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.";
|
||||
FunctionDocumentation::Examples examples = {{"simple", "SELECT dateTime64ToSnowflakeID(toDateTime64('2021-08-15 18:57:56', 3, 'Asia/Shanghai'))", "6832626394434895872"}};
|
||||
FunctionDocumentation::Categories categories = {"Snowflake ID"};
|
||||
|
||||
factory.registerFunction<FunctionDateTime64ToSnowflakeID>({description, syntax, arguments, returned_value, examples, categories});
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -207,7 +207,7 @@ public:
|
||||
|
||||
REGISTER_FUNCTION(GenerateSnowflakeID)
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Generates a Snowflake ID. The generated Snowflake ID contains the current Unix timestamp in milliseconds 41 (+ 1 top zero bit) bits, followed by machine id (10 bits), a counter (12 bits) to distinguish IDs within a millisecond. For any given timestamp (unix_ts_ms), the counter starts at 0 and is incremented by 1 for each new Snowflake ID until the timestamp changes. In case the counter overflows, the timestamp field is incremented by 1 and the counter is reset to 0. Function generateSnowflakeID guarantees that the counter field within a timestamp increments monotonically across all function invocations in concurrently running threads and queries.)";
|
||||
FunctionDocumentation::Description description = R"(Generates a Snowflake ID. The generated Snowflake ID contains the current Unix timestamp in milliseconds (41 + 1 top zero bits), followed by a machine id (10 bits), and a counter (12 bits) to distinguish IDs within a millisecond. For any given timestamp (unix_ts_ms), the counter starts at 0 and is incremented by 1 for each new Snowflake ID until the timestamp changes. In case the counter overflows, the timestamp field is incremented by 1 and the counter is reset to 0. Function generateSnowflakeID guarantees that the counter field within a timestamp increments monotonically across all function invocations in concurrently running threads and queries.)";
|
||||
FunctionDocumentation::Syntax syntax = "generateSnowflakeID([expression])";
|
||||
FunctionDocumentation::Arguments arguments = {{"expression", "The expression is used to bypass common subexpression elimination if the function is called multiple times in a query but otherwise ignored. Optional."}};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "A value of type UInt64";
|
||||
|
||||
@ -11,11 +11,17 @@
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
|
||||
/// ------------------------------------------------------------------------------------------------------------------------------
|
||||
/// The functions in this file are deprecated and should be removed in favor of functions 'snowflakeIDToDateTime[64]' and
|
||||
/// 'dateTime[64]ToSnowflakeID' by summer 2025. Please also mark setting `uniform_snowflake_conversion_functions` as obsolete then.
|
||||
/// ------------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int DEPRECATED_FUNCTION;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -34,10 +40,19 @@ constexpr int time_shift = 22;
|
||||
class FunctionDateTimeToSnowflake : public IFunction
|
||||
{
|
||||
private:
|
||||
const char * name;
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
explicit FunctionDateTimeToSnowflake(const char * name_) : name(name_) { }
|
||||
static constexpr auto name = "dateTimeToSnowflake";
|
||||
|
||||
static FunctionPtr create(ContextPtr context)
|
||||
{
|
||||
return std::make_shared<FunctionDateTimeToSnowflake>(context);
|
||||
}
|
||||
|
||||
explicit FunctionDateTimeToSnowflake(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
@ -56,6 +71,9 @@ public:
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated, to enable it disable setting 'uniform_snowflake_conversion_functions'", getName());
|
||||
|
||||
const auto & src = arguments[0];
|
||||
const auto & src_column = *src.column;
|
||||
|
||||
@ -73,13 +91,20 @@ public:
|
||||
class FunctionSnowflakeToDateTime : public IFunction
|
||||
{
|
||||
private:
|
||||
const char * name;
|
||||
const bool allow_nonconst_timezone_arguments;
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
explicit FunctionSnowflakeToDateTime(const char * name_, ContextPtr context)
|
||||
: name(name_)
|
||||
, allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments)
|
||||
static constexpr auto name = "snowflakeToDateTime";
|
||||
|
||||
static FunctionPtr create(ContextPtr context)
|
||||
{
|
||||
return std::make_shared<FunctionSnowflakeToDateTime>(context);
|
||||
}
|
||||
|
||||
explicit FunctionSnowflakeToDateTime(ContextPtr context)
|
||||
: allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments)
|
||||
, uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
@ -107,6 +132,9 @@ public:
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated, to enable it disable setting 'uniform_snowflake_conversion_functions'", getName());
|
||||
|
||||
const auto & src = arguments[0];
|
||||
const auto & src_column = *src.column;
|
||||
|
||||
@ -138,10 +166,19 @@ public:
|
||||
class FunctionDateTime64ToSnowflake : public IFunction
|
||||
{
|
||||
private:
|
||||
const char * name;
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
explicit FunctionDateTime64ToSnowflake(const char * name_) : name(name_) { }
|
||||
static constexpr auto name = "dateTime64ToSnowflake";
|
||||
|
||||
static FunctionPtr create(ContextPtr context)
|
||||
{
|
||||
return std::make_shared<FunctionDateTime64ToSnowflake>(context);
|
||||
}
|
||||
|
||||
explicit FunctionDateTime64ToSnowflake(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
@ -160,6 +197,9 @@ public:
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated, to enable it disable setting 'uniform_snowflake_conversion_functions'", getName());
|
||||
|
||||
const auto & src = arguments[0];
|
||||
|
||||
const auto & src_column = *src.column;
|
||||
@ -185,13 +225,20 @@ public:
|
||||
class FunctionSnowflakeToDateTime64 : public IFunction
|
||||
{
|
||||
private:
|
||||
const char * name;
|
||||
const bool allow_nonconst_timezone_arguments;
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
explicit FunctionSnowflakeToDateTime64(const char * name_, ContextPtr context)
|
||||
: name(name_)
|
||||
, allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments)
|
||||
static constexpr auto name = "snowflakeToDateTime64";
|
||||
|
||||
static FunctionPtr create(ContextPtr context)
|
||||
{
|
||||
return std::make_shared<FunctionSnowflakeToDateTime64>(context);
|
||||
}
|
||||
|
||||
explicit FunctionSnowflakeToDateTime64(ContextPtr context)
|
||||
: allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments)
|
||||
, uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
@ -219,6 +266,9 @@ public:
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated, to enable it disable setting 'uniform_snowflake_conversion_functions'", getName());
|
||||
|
||||
const auto & src = arguments[0];
|
||||
const auto & src_column = *src.column;
|
||||
|
||||
@ -246,27 +296,12 @@ public:
|
||||
|
||||
}
|
||||
|
||||
REGISTER_FUNCTION(DateTimeToSnowflake)
|
||||
REGISTER_FUNCTION(LegacySnowflakeConversion)
|
||||
{
|
||||
factory.registerFunction("dateTimeToSnowflake",
|
||||
[](ContextPtr){ return std::make_shared<FunctionDateTimeToSnowflake>("dateTimeToSnowflake"); });
|
||||
}
|
||||
|
||||
REGISTER_FUNCTION(DateTime64ToSnowflake)
|
||||
{
|
||||
factory.registerFunction("dateTime64ToSnowflake",
|
||||
[](ContextPtr){ return std::make_shared<FunctionDateTime64ToSnowflake>("dateTime64ToSnowflake"); });
|
||||
}
|
||||
|
||||
REGISTER_FUNCTION(SnowflakeToDateTime)
|
||||
{
|
||||
factory.registerFunction("snowflakeToDateTime",
|
||||
[](ContextPtr context){ return std::make_shared<FunctionSnowflakeToDateTime>("snowflakeToDateTime", context); });
|
||||
}
|
||||
REGISTER_FUNCTION(SnowflakeToDateTime64)
|
||||
{
|
||||
factory.registerFunction("snowflakeToDateTime64",
|
||||
[](ContextPtr context){ return std::make_shared<FunctionSnowflakeToDateTime64>("snowflakeToDateTime64", context); });
|
||||
factory.registerFunction<FunctionSnowflakeToDateTime>();
|
||||
factory.registerFunction<FunctionSnowflakeToDateTime64>();
|
||||
factory.registerFunction<FunctionDateTimeToSnowflake>();
|
||||
factory.registerFunction<FunctionDateTime64ToSnowflake>();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
217
src/Functions/snowflakeIDToDateTime.cpp
Normal file
217
src/Functions/snowflakeIDToDateTime.cpp
Normal file
@ -0,0 +1,217 @@
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/extractTimeZoneFromFunctionArguments.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Columns/ColumnsDateTime.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int UNKNOWN_FUNCTION;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/// See generateSnowflakeID.cpp
|
||||
constexpr int time_shift = 22;
|
||||
|
||||
}
|
||||
|
||||
class FunctionSnowflakeIDToDateTime : public IFunction
|
||||
{
|
||||
private:
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
const bool allow_nonconst_timezone_arguments;
|
||||
|
||||
public:
|
||||
static constexpr auto name = "snowflakeIDToDateTime";
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionSnowflakeIDToDateTime>(context); }
|
||||
explicit FunctionSnowflakeIDToDateTime(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
, allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool isVariadic() const override { return true; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isUInt64), nullptr, "UInt64"}
|
||||
};
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"epoch", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), isColumnConst, "UInt*"},
|
||||
{"time_zone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args, optional_args);
|
||||
|
||||
String timezone;
|
||||
if (arguments.size() == 3)
|
||||
timezone = extractTimeZoneNameFromFunctionArguments(arguments, 2, 0, allow_nonconst_timezone_arguments);
|
||||
|
||||
return std::make_shared<DataTypeDateTime>(timezone);
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (!uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "To use function {}, setting 'uniform_snowflake_conversion_functions' must be enabled", getName());
|
||||
|
||||
const auto & col_src = *arguments[0].column;
|
||||
|
||||
size_t epoch = 0;
|
||||
if (arguments.size() >= 2 && input_rows_count != 0)
|
||||
{
|
||||
const auto & col_epoch = *arguments[1].column;
|
||||
epoch = col_epoch.getUInt(0);
|
||||
}
|
||||
|
||||
auto col_res = ColumnDateTime::create(input_rows_count);
|
||||
auto & res_data = col_res->getData();
|
||||
|
||||
if (const auto * col_src_non_const = typeid_cast<const ColumnUInt64 *>(&col_src))
|
||||
{
|
||||
const auto & src_data = col_src_non_const->getData();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = static_cast<UInt32>(((src_data[i] >> time_shift) + epoch) / 1000);
|
||||
}
|
||||
else if (const auto * col_src_const = typeid_cast<const ColumnConst *>(&col_src))
|
||||
{
|
||||
UInt64 src_val = col_src_const->getValue<UInt64>();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = static_cast<UInt32>(((src_val >> time_shift) + epoch) / 1000);
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal argument for function {}", name);
|
||||
|
||||
return col_res;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class FunctionSnowflakeIDToDateTime64 : public IFunction
|
||||
{
|
||||
private:
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
const bool allow_nonconst_timezone_arguments;
|
||||
|
||||
public:
|
||||
static constexpr auto name = "snowflakeIDToDateTime64";
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionSnowflakeIDToDateTime64>(context); }
|
||||
explicit FunctionSnowflakeIDToDateTime64(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
, allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool isVariadic() const override { return true; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isUInt64), nullptr, "UInt64"}
|
||||
};
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"epoch", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), isColumnConst, "UInt*"},
|
||||
{"time_zone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args, optional_args);
|
||||
|
||||
String timezone;
|
||||
if (arguments.size() == 3)
|
||||
timezone = extractTimeZoneNameFromFunctionArguments(arguments, 2, 0, allow_nonconst_timezone_arguments);
|
||||
|
||||
return std::make_shared<DataTypeDateTime64>(3, timezone);
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (!uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "To use function {}, setting 'uniform_snowflake_conversion_functions' must be enabled", getName());
|
||||
|
||||
const auto & col_src = *arguments[0].column;
|
||||
|
||||
size_t epoch = 0;
|
||||
if (arguments.size() >= 2 && input_rows_count != 0)
|
||||
{
|
||||
const auto & col_epoch = *arguments[1].column;
|
||||
epoch = col_epoch.getUInt(0);
|
||||
}
|
||||
|
||||
auto col_res = ColumnDateTime64::create(input_rows_count, 3);
|
||||
auto & res_data = col_res->getData();
|
||||
|
||||
if (const auto * col_src_non_const = typeid_cast<const ColumnUInt64 *>(&col_src))
|
||||
{
|
||||
const auto & src_data = col_src_non_const->getData();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = (src_data[i] >> time_shift) + epoch;
|
||||
}
|
||||
else if (const auto * col_src_const = typeid_cast<const ColumnConst *>(&col_src))
|
||||
{
|
||||
UInt64 src_val = col_src_const->getValue<UInt64>();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = (src_val >> time_shift) + epoch;
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal argument for function {}", name);
|
||||
|
||||
return col_res;
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_FUNCTION(SnowflakeIDToDateTime)
|
||||
{
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime](../data-types/datetime.md).)";
|
||||
FunctionDocumentation::Syntax syntax = "snowflakeIDToDateTime(value[, epoch[, time_zone]])";
|
||||
FunctionDocumentation::Arguments arguments = {
|
||||
{"value", "Snowflake ID. [UInt64](../data-types/int-uint.md)"},
|
||||
{"epoch", "Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md)"},
|
||||
{"time_zone", "[Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md)"}
|
||||
};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "The timestamp component of `value` as a [DateTime](../data-types/datetime.md) value.";
|
||||
FunctionDocumentation::Examples examples = {{"simple", "SELECT snowflakeIDToDateTime(7204436857747984384)", "2024-06-06 10:59:58"}};
|
||||
FunctionDocumentation::Categories categories = {"Snowflake ID"};
|
||||
|
||||
factory.registerFunction<FunctionSnowflakeIDToDateTime>({description, syntax, arguments, returned_value, examples, categories});
|
||||
}
|
||||
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime64](../data-types/datetime64.md).)";
|
||||
FunctionDocumentation::Syntax syntax = "snowflakeIDToDateTime64(value[, epoch[, time_zone]])";
|
||||
FunctionDocumentation::Arguments arguments = {
|
||||
{"value", "Snowflake ID. [UInt64](../data-types/int-uint.md)"},
|
||||
{"epoch", "Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md)"},
|
||||
{"time_zone", "[Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md)"}
|
||||
};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "The timestamp component of `value` as a [DateTime64](../data-types/datetime64.md) with scale = 3, i.e. millisecond precision.";
|
||||
FunctionDocumentation::Examples examples = {{"simple", "SELECT snowflakeIDToDateTime64(7204436857747984384)", "2024-06-06 10:59:58"}};
|
||||
FunctionDocumentation::Categories categories = {"Snowflake ID"};
|
||||
|
||||
factory.registerFunction<FunctionSnowflakeIDToDateTime64>({description, syntax, arguments, returned_value, examples, categories});
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -30,10 +30,6 @@
|
||||
#include <base/sleep.h>
|
||||
|
||||
|
||||
#ifdef ADDRESS_SANITIZER
|
||||
#include <sanitizer/lsan_interface.h>
|
||||
#endif
|
||||
|
||||
namespace ProfileEvents
|
||||
{
|
||||
extern const Event S3WriteRequestsErrors;
|
||||
@ -880,14 +876,7 @@ void ClientCacheRegistry::clearCacheForAll()
|
||||
ClientFactory::ClientFactory()
|
||||
{
|
||||
aws_options = Aws::SDKOptions{};
|
||||
{
|
||||
#ifdef ADDRESS_SANITIZER
|
||||
/// Leak sanitizer (part of address sanitizer) thinks that memory in OpenSSL (called by AWS SDK) is allocated but not
|
||||
/// released. Actually, the memory is released at the end of the program (ClientFactory is a singleton, see the dtor).
|
||||
__lsan::ScopedDisabler lsan_disabler;
|
||||
#endif
|
||||
Aws::InitAPI(aws_options);
|
||||
}
|
||||
Aws::InitAPI(aws_options);
|
||||
Aws::Utils::Logging::InitializeAWSLogging(std::make_shared<AWSLogger>(false));
|
||||
Aws::Http::SetHttpClientFactory(std::make_shared<PocoHTTPClientFactory>());
|
||||
}
|
||||
|
||||
@ -118,7 +118,7 @@ namespace
|
||||
/// Checks if the current user has enough access rights granted with grant option to grant or revoke specified access rights.
|
||||
void checkGrantOption(
|
||||
const AccessControl & access_control,
|
||||
const ContextAccess & current_user_access,
|
||||
const ContextAccessWrapper & current_user_access,
|
||||
const std::vector<UUID> & grantees_from_query,
|
||||
bool & need_check_grantees_are_allowed,
|
||||
const AccessRightsElements & elements_to_grant,
|
||||
@ -200,7 +200,7 @@ namespace
|
||||
/// Checks if the current user has enough roles granted with admin option to grant or revoke specified roles.
|
||||
void checkAdminOption(
|
||||
const AccessControl & access_control,
|
||||
const ContextAccess & current_user_access,
|
||||
const ContextAccessWrapper & current_user_access,
|
||||
const std::vector<UUID> & grantees_from_query,
|
||||
bool & need_check_grantees_are_allowed,
|
||||
const std::vector<UUID> & roles_to_grant,
|
||||
@ -277,7 +277,7 @@ namespace
|
||||
/// This function is less accurate than checkAdminOption() because it cannot use any information about
|
||||
/// granted roles the grantees currently have (due to those grantees are located on multiple nodes,
|
||||
/// we just don't have the full information about them).
|
||||
void checkAdminOptionForExecutingOnCluster(const ContextAccess & current_user_access,
|
||||
void checkAdminOptionForExecutingOnCluster(const ContextAccessWrapper & current_user_access,
|
||||
const std::vector<UUID> roles_to_grant,
|
||||
const RolesOrUsersSet & roles_to_revoke)
|
||||
{
|
||||
@ -376,7 +376,7 @@ namespace
|
||||
/// Calculates all available rights to grant with current user intersection.
|
||||
void calculateCurrentGrantRightsWithIntersection(
|
||||
AccessRights & rights,
|
||||
std::shared_ptr<const ContextAccess> current_user_access,
|
||||
std::shared_ptr<const ContextAccessWrapper> current_user_access,
|
||||
const AccessRightsElements & elements_to_grant)
|
||||
{
|
||||
AccessRightsElements current_user_grantable_elements;
|
||||
|
||||
@ -786,9 +786,6 @@ Block ActionsDAG::updateHeader(const Block & header) const
|
||||
for (auto & col : result_columns)
|
||||
res.insert(std::move(col));
|
||||
|
||||
if (isInputProjected())
|
||||
return res;
|
||||
|
||||
res.reserve(header.columns() - pos_to_remove.size());
|
||||
for (size_t i = 0; i < header.columns(); i++)
|
||||
{
|
||||
@ -1150,8 +1147,33 @@ void ActionsDAG::project(const NamesWithAliases & projection)
|
||||
}
|
||||
|
||||
removeUnusedActions();
|
||||
projectInput();
|
||||
projected_output = true;
|
||||
}
|
||||
|
||||
void ActionsDAG::appendInputsForUnusedColumns(const Block & sample_block)
|
||||
{
|
||||
std::unordered_map<std::string_view, std::list<size_t>> names_map;
|
||||
size_t num_columns = sample_block.columns();
|
||||
for (size_t pos = 0; pos < num_columns; ++pos)
|
||||
names_map[sample_block.getByPosition(pos).name].push_back(pos);
|
||||
|
||||
for (const auto * input : inputs)
|
||||
{
|
||||
auto & positions = names_map[input->result_name];
|
||||
if (positions.empty())
|
||||
throw Exception(ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK,
|
||||
"Not found column {} in block {}", input->result_name, sample_block.dumpStructure());
|
||||
|
||||
positions.pop_front();
|
||||
}
|
||||
|
||||
for (const auto & [_, positions] : names_map)
|
||||
{
|
||||
for (auto pos : positions)
|
||||
{
|
||||
const auto & col = sample_block.getByPosition(pos);
|
||||
addInput(col.name, col.type);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool ActionsDAG::tryRestoreColumn(const std::string & column_name)
|
||||
@ -1227,8 +1249,6 @@ bool ActionsDAG::removeUnusedResult(const std::string & column_name)
|
||||
ActionsDAGPtr ActionsDAG::clone() const
|
||||
{
|
||||
auto actions = std::make_shared<ActionsDAG>();
|
||||
actions->project_input = project_input;
|
||||
actions->projected_output = projected_output;
|
||||
|
||||
std::unordered_map<const Node *, Node *> copy_map;
|
||||
|
||||
@ -1322,9 +1342,6 @@ std::string ActionsDAG::dumpDAG() const
|
||||
out << ' ' << map[node];
|
||||
out << '\n';
|
||||
|
||||
out << "Project input: " << project_input << '\n';
|
||||
out << "Projected output: " << projected_output << '\n';
|
||||
|
||||
return out.str();
|
||||
}
|
||||
|
||||
@ -1409,7 +1426,7 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
|
||||
|
||||
FunctionOverloadResolverPtr func_builder_materialize = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionMaterialize>());
|
||||
|
||||
std::map<std::string_view, std::list<size_t>> inputs;
|
||||
std::unordered_map<std::string_view, std::list<size_t>> inputs;
|
||||
if (mode == MatchColumnsMode::Name)
|
||||
{
|
||||
size_t input_nodes_size = actions_dag->inputs.size();
|
||||
@ -1525,8 +1542,7 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
|
||||
}
|
||||
|
||||
actions_dag->outputs.swap(projection);
|
||||
actions_dag->removeUnusedActions();
|
||||
actions_dag->projectInput();
|
||||
actions_dag->removeUnusedActions(false);
|
||||
|
||||
return actions_dag;
|
||||
}
|
||||
@ -1584,10 +1600,6 @@ void ActionsDAG::mergeInplace(ActionsDAG && second)
|
||||
auto it = first_result.find(input_node->result_name);
|
||||
if (it == first_result.end() || it->second.empty())
|
||||
{
|
||||
if (first.project_input)
|
||||
throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER,
|
||||
"Cannot find column {} in ActionsDAG result", input_node->result_name);
|
||||
|
||||
first.inputs.push_back(input_node);
|
||||
}
|
||||
else
|
||||
@ -1623,13 +1635,6 @@ void ActionsDAG::mergeInplace(ActionsDAG && second)
|
||||
}
|
||||
}
|
||||
|
||||
/// Update output nodes.
|
||||
if (second.project_input)
|
||||
{
|
||||
first.outputs.swap(second.outputs);
|
||||
first.project_input = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
/// Add not removed result from first actions.
|
||||
for (const auto * output_node : first.outputs)
|
||||
@ -1645,8 +1650,6 @@ void ActionsDAG::mergeInplace(ActionsDAG && second)
|
||||
}
|
||||
|
||||
first.nodes.splice(first.nodes.end(), std::move(second.nodes));
|
||||
|
||||
first.projected_output = second.projected_output;
|
||||
}
|
||||
|
||||
void ActionsDAG::mergeNodes(ActionsDAG && second, NodeRawConstPtrs * out_outputs)
|
||||
@ -2042,7 +2045,6 @@ ActionsDAG::SplitResult ActionsDAG::splitActionsBeforeArrayJoin(const NameSet &
|
||||
}
|
||||
|
||||
auto res = split(split_nodes);
|
||||
res.second->project_input = project_input;
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -2086,7 +2088,6 @@ ActionsDAG::SplitResult ActionsDAG::splitActionsBySortingDescription(const NameS
|
||||
dumpDAG());
|
||||
|
||||
auto res = split(split_nodes);
|
||||
res.second->project_input = project_input;
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -2158,7 +2159,6 @@ ActionsDAG::SplitResult ActionsDAG::splitActionsForFilter(const std::string & co
|
||||
|
||||
std::unordered_set<const Node *> split_nodes = {node};
|
||||
auto res = split(split_nodes);
|
||||
res.second->project_input = project_input;
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -2745,11 +2745,7 @@ void ActionsDAG::removeUnusedConjunctions(NodeRawConstPtrs rejected_conjunctions
|
||||
|
||||
std::unordered_set<const Node *> used_inputs;
|
||||
for (const auto * input : inputs)
|
||||
{
|
||||
if (removes_filter && input == predicate)
|
||||
continue;
|
||||
used_inputs.insert(input);
|
||||
}
|
||||
|
||||
removeUnusedActions(used_inputs);
|
||||
}
|
||||
|
||||
@ -103,13 +103,11 @@ private:
|
||||
NodeRawConstPtrs inputs;
|
||||
NodeRawConstPtrs outputs;
|
||||
|
||||
bool project_input = false;
|
||||
bool projected_output = false;
|
||||
|
||||
public:
|
||||
ActionsDAG() = default;
|
||||
ActionsDAG(ActionsDAG &&) = default;
|
||||
ActionsDAG(const ActionsDAG &) = delete;
|
||||
ActionsDAG & operator=(ActionsDAG &&) = default;
|
||||
ActionsDAG & operator=(const ActionsDAG &) = delete;
|
||||
explicit ActionsDAG(const NamesAndTypesList & inputs_);
|
||||
explicit ActionsDAG(const ColumnsWithTypeAndName & inputs_);
|
||||
@ -168,9 +166,12 @@ public:
|
||||
/// Call addAlias several times.
|
||||
void addAliases(const NamesWithAliases & aliases);
|
||||
|
||||
/// Add alias actions and remove unused columns from outputs. Also specify result columns order in outputs.
|
||||
/// Add alias actions. Also specify result columns order in outputs.
|
||||
void project(const NamesWithAliases & projection);
|
||||
|
||||
/// Add input for every column from sample_block which is not mapped to existing input.
|
||||
void appendInputsForUnusedColumns(const Block & sample_block);
|
||||
|
||||
/// If column is not in outputs, try to find it in nodes and insert back into outputs.
|
||||
bool tryRestoreColumn(const std::string & column_name);
|
||||
|
||||
@ -179,10 +180,6 @@ public:
|
||||
/// Return true if column was removed from inputs.
|
||||
bool removeUnusedResult(const std::string & column_name);
|
||||
|
||||
void projectInput(bool project = true) { project_input = project; }
|
||||
bool isInputProjected() const { return project_input; }
|
||||
bool isOutputProjected() const { return projected_output; }
|
||||
|
||||
/// Remove actions that are not needed to compute output nodes
|
||||
void removeUnusedActions(bool allow_remove_inputs = true, bool allow_constant_folding = true);
|
||||
|
||||
@ -510,4 +507,15 @@ struct ActionDAGNodes
|
||||
ActionsDAG::NodeRawConstPtrs nodes;
|
||||
};
|
||||
|
||||
/// Helper for query analysis.
|
||||
/// If project_input is set, all columns not found in inputs should be removed.
|
||||
/// Now, we do it before adding a step to query plan by calling appendInputsForUnusedColumns.
|
||||
struct ActionsAndProjectInputsFlag
|
||||
{
|
||||
ActionsDAG dag;
|
||||
bool project_input = false;
|
||||
};
|
||||
|
||||
using ActionsAndProjectInputsFlagPtr = std::shared_ptr<ActionsAndProjectInputsFlag>;
|
||||
|
||||
}
|
||||
|
||||
@ -405,6 +405,9 @@ Block createBlockForSet(
|
||||
|
||||
}
|
||||
|
||||
ScopeStack::Level::Level() = default;
|
||||
ScopeStack::Level::~Level() = default;
|
||||
ScopeStack::Level::Level(Level &&) noexcept = default;
|
||||
|
||||
FutureSetPtr makeExplicitSet(
|
||||
const ASTFunction * node, const ActionsDAG & actions, ContextPtr context, PreparedSets & prepared_sets)
|
||||
@ -499,16 +502,12 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
ScopeStack::Level::~Level() = default;
|
||||
ScopeStack::Level::Level() = default;
|
||||
ScopeStack::Level::Level(Level &&) noexcept = default;
|
||||
|
||||
ActionsMatcher::Data::Data(
|
||||
ContextPtr context_,
|
||||
SizeLimits set_size_limit_,
|
||||
size_t subquery_depth_,
|
||||
std::reference_wrapper<const NamesAndTypesList> source_columns_,
|
||||
ActionsDAGPtr actions_dag,
|
||||
ActionsDAG actions_dag,
|
||||
PreparedSetsPtr prepared_sets_,
|
||||
bool no_subqueries_,
|
||||
bool no_makeset_,
|
||||
@ -544,13 +543,13 @@ std::vector<std::string_view> ActionsMatcher::Data::getAllColumnNames() const
|
||||
return index.getAllNames();
|
||||
}
|
||||
|
||||
ScopeStack::ScopeStack(ActionsDAGPtr actions_dag, ContextPtr context_) : WithContext(context_)
|
||||
ScopeStack::ScopeStack(ActionsDAG actions_dag, ContextPtr context_) : WithContext(context_)
|
||||
{
|
||||
auto & level = stack.emplace_back();
|
||||
level.actions_dag = std::move(actions_dag);
|
||||
level.index = std::make_unique<ScopeStack::Index>(level.actions_dag->getOutputs());
|
||||
level.index = std::make_unique<ScopeStack::Index>(level.actions_dag.getOutputs());
|
||||
|
||||
for (const auto & node : level.actions_dag->getOutputs())
|
||||
for (const auto & node : level.actions_dag.getOutputs())
|
||||
if (node->type == ActionsDAG::ActionType::INPUT)
|
||||
level.inputs.emplace(node->result_name);
|
||||
}
|
||||
@ -558,22 +557,21 @@ ScopeStack::ScopeStack(ActionsDAGPtr actions_dag, ContextPtr context_) : WithCon
|
||||
void ScopeStack::pushLevel(const NamesAndTypesList & input_columns)
|
||||
{
|
||||
auto & level = stack.emplace_back();
|
||||
level.actions_dag = std::make_shared<ActionsDAG>();
|
||||
level.index = std::make_unique<ScopeStack::Index>(level.actions_dag->getOutputs());
|
||||
level.index = std::make_unique<ScopeStack::Index>(level.actions_dag.getOutputs());
|
||||
const auto & prev = stack[stack.size() - 2];
|
||||
|
||||
for (const auto & input_column : input_columns)
|
||||
{
|
||||
const auto & node = level.actions_dag->addInput(input_column.name, input_column.type);
|
||||
const auto & node = level.actions_dag.addInput(input_column.name, input_column.type);
|
||||
level.index->addNode(&node);
|
||||
level.inputs.emplace(input_column.name);
|
||||
}
|
||||
|
||||
for (const auto & node : prev.actions_dag->getOutputs())
|
||||
for (const auto & node : prev.actions_dag.getOutputs())
|
||||
{
|
||||
if (!level.index->contains(node->result_name))
|
||||
{
|
||||
const auto & input = level.actions_dag->addInput({node->column, node->result_type, node->result_name});
|
||||
const auto & input = level.actions_dag.addInput({node->column, node->result_type, node->result_name});
|
||||
level.index->addNode(&input);
|
||||
}
|
||||
}
|
||||
@ -598,12 +596,12 @@ size_t ScopeStack::getColumnLevel(const std::string & name)
|
||||
|
||||
void ScopeStack::addColumn(ColumnWithTypeAndName column)
|
||||
{
|
||||
const auto & node = stack[0].actions_dag->addColumn(std::move(column));
|
||||
const auto & node = stack[0].actions_dag.addColumn(std::move(column));
|
||||
stack[0].index->addNode(&node);
|
||||
|
||||
for (size_t j = 1; j < stack.size(); ++j)
|
||||
{
|
||||
const auto & input = stack[j].actions_dag->addInput({node.column, node.result_type, node.result_name});
|
||||
const auto & input = stack[j].actions_dag.addInput({node.column, node.result_type, node.result_name});
|
||||
stack[j].index->addNode(&input);
|
||||
}
|
||||
}
|
||||
@ -612,12 +610,12 @@ void ScopeStack::addAlias(const std::string & name, std::string alias)
|
||||
{
|
||||
auto level = getColumnLevel(name);
|
||||
const auto & source = stack[level].index->getNode(name);
|
||||
const auto & node = stack[level].actions_dag->addAlias(source, std::move(alias));
|
||||
const auto & node = stack[level].actions_dag.addAlias(source, std::move(alias));
|
||||
stack[level].index->addNode(&node);
|
||||
|
||||
for (size_t j = level + 1; j < stack.size(); ++j)
|
||||
{
|
||||
const auto & input = stack[j].actions_dag->addInput({node.column, node.result_type, node.result_name});
|
||||
const auto & input = stack[j].actions_dag.addInput({node.column, node.result_type, node.result_name});
|
||||
stack[j].index->addNode(&input);
|
||||
}
|
||||
}
|
||||
@ -631,12 +629,12 @@ void ScopeStack::addArrayJoin(const std::string & source_name, std::string resul
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expression with arrayJoin cannot depend on lambda argument: {}",
|
||||
source_name);
|
||||
|
||||
const auto & node = stack.front().actions_dag->addArrayJoin(*source_node, std::move(result_name));
|
||||
const auto & node = stack.front().actions_dag.addArrayJoin(*source_node, std::move(result_name));
|
||||
stack.front().index->addNode(&node);
|
||||
|
||||
for (size_t j = 1; j < stack.size(); ++j)
|
||||
{
|
||||
const auto & input = stack[j].actions_dag->addInput({node.column, node.result_type, node.result_name});
|
||||
const auto & input = stack[j].actions_dag.addInput({node.column, node.result_type, node.result_name});
|
||||
stack[j].index->addNode(&input);
|
||||
}
|
||||
}
|
||||
@ -655,17 +653,17 @@ void ScopeStack::addFunction(
|
||||
for (const auto & argument : argument_names)
|
||||
children.push_back(&stack[level].index->getNode(argument));
|
||||
|
||||
const auto & node = stack[level].actions_dag->addFunction(function, std::move(children), std::move(result_name));
|
||||
const auto & node = stack[level].actions_dag.addFunction(function, std::move(children), std::move(result_name));
|
||||
stack[level].index->addNode(&node);
|
||||
|
||||
for (size_t j = level + 1; j < stack.size(); ++j)
|
||||
{
|
||||
const auto & input = stack[j].actions_dag->addInput({node.column, node.result_type, node.result_name});
|
||||
const auto & input = stack[j].actions_dag.addInput({node.column, node.result_type, node.result_name});
|
||||
stack[j].index->addNode(&input);
|
||||
}
|
||||
}
|
||||
|
||||
ActionsDAGPtr ScopeStack::popLevel()
|
||||
ActionsDAG ScopeStack::popLevel()
|
||||
{
|
||||
auto res = std::move(stack.back().actions_dag);
|
||||
stack.pop_back();
|
||||
@ -674,12 +672,12 @@ ActionsDAGPtr ScopeStack::popLevel()
|
||||
|
||||
std::string ScopeStack::dumpNames() const
|
||||
{
|
||||
return stack.back().actions_dag->dumpNames();
|
||||
return stack.back().actions_dag.dumpNames();
|
||||
}
|
||||
|
||||
const ActionsDAG & ScopeStack::getLastActions() const
|
||||
{
|
||||
return *stack.back().actions_dag;
|
||||
return stack.back().actions_dag;
|
||||
}
|
||||
|
||||
const ScopeStack::Index & ScopeStack::getLastActionsIndex() const
|
||||
@ -1002,7 +1000,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
|
||||
data.set_size_limit,
|
||||
data.subquery_depth,
|
||||
data.source_columns,
|
||||
std::make_shared<ActionsDAG>(data.source_columns),
|
||||
ActionsDAG(data.source_columns),
|
||||
data.prepared_sets,
|
||||
data.no_subqueries,
|
||||
data.no_makeset,
|
||||
@ -1021,10 +1019,10 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
|
||||
}
|
||||
|
||||
auto dag = index_hint_data.getActions();
|
||||
dag->project(args);
|
||||
dag.project(args);
|
||||
|
||||
auto index_hint = std::make_shared<FunctionIndexHint>();
|
||||
index_hint->setActions(std::move(dag));
|
||||
index_hint->setActions(std::make_shared<ActionsDAG>(std::move(dag)));
|
||||
|
||||
// Arguments are removed. We add function instead of constant column to avoid constant folding.
|
||||
data.addFunction(std::make_unique<FunctionToOverloadResolverAdaptor>(index_hint), {}, column_name);
|
||||
@ -1284,10 +1282,10 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
|
||||
auto lambda_dag = data.actions_stack.popLevel();
|
||||
|
||||
String result_name = lambda->arguments->children.at(1)->getColumnName();
|
||||
lambda_dag->removeUnusedActions(Names(1, result_name));
|
||||
lambda_dag.removeUnusedActions(Names(1, result_name));
|
||||
|
||||
auto lambda_actions = std::make_shared<ExpressionActions>(
|
||||
lambda_dag,
|
||||
std::make_shared<ActionsDAG>(std::move(lambda_dag)),
|
||||
ExpressionActionsSettings::fromContext(data.getContext(), CompileExpressions::yes));
|
||||
|
||||
DataTypePtr result_type = lambda_actions->getSampleBlock().getByName(result_name).type;
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <deque>
|
||||
#include <string_view>
|
||||
#include <Core/ColumnNumbers.h>
|
||||
#include <Core/ColumnWithTypeAndName.h>
|
||||
@ -9,6 +10,7 @@
|
||||
#include <Interpreters/PreparedSets.h>
|
||||
#include <Parsers/IAST.h>
|
||||
#include <QueryPipeline/SizeLimits.h>
|
||||
#include <Interpreters/ActionsDAG.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -43,20 +45,20 @@ struct ScopeStack : WithContext
|
||||
|
||||
struct Level
|
||||
{
|
||||
ActionsDAGPtr actions_dag;
|
||||
ActionsDAG actions_dag;
|
||||
IndexPtr index;
|
||||
NameSet inputs;
|
||||
|
||||
~Level();
|
||||
Level();
|
||||
Level(Level &&) noexcept;
|
||||
~Level();
|
||||
};
|
||||
|
||||
using Levels = std::vector<Level>;
|
||||
using Levels = std::deque<Level>;
|
||||
|
||||
Levels stack;
|
||||
|
||||
ScopeStack(ActionsDAGPtr actions_dag, ContextPtr context_);
|
||||
ScopeStack(ActionsDAG actions_dag, ContextPtr context_);
|
||||
|
||||
void pushLevel(const NamesAndTypesList & input_columns);
|
||||
|
||||
@ -67,7 +69,7 @@ struct ScopeStack : WithContext
|
||||
void addArrayJoin(const std::string & source_name, std::string result_name);
|
||||
void addFunction(const FunctionOverloadResolverPtr & function, const Names & argument_names, std::string result_name);
|
||||
|
||||
ActionsDAGPtr popLevel();
|
||||
ActionsDAG popLevel();
|
||||
|
||||
const ActionsDAG & getLastActions() const;
|
||||
const Index & getLastActionsIndex() const;
|
||||
@ -147,7 +149,7 @@ public:
|
||||
SizeLimits set_size_limit_,
|
||||
size_t subquery_depth_,
|
||||
std::reference_wrapper<const NamesAndTypesList> source_columns_,
|
||||
ActionsDAGPtr actions_dag,
|
||||
ActionsDAG actions_dag,
|
||||
PreparedSetsPtr prepared_sets_,
|
||||
bool no_subqueries_,
|
||||
bool no_makeset_,
|
||||
@ -182,7 +184,7 @@ public:
|
||||
actions_stack.addFunction(function, argument_names, std::move(result_name));
|
||||
}
|
||||
|
||||
ActionsDAGPtr getActions()
|
||||
ActionsDAG getActions()
|
||||
{
|
||||
return actions_stack.popLevel();
|
||||
}
|
||||
|
||||
@ -835,6 +835,7 @@ ContextMutablePtr Context::createGlobal(ContextSharedPart * shared_part)
|
||||
auto res = std::shared_ptr<Context>(new Context);
|
||||
res->shared = shared_part;
|
||||
res->query_access_info = std::make_shared<QueryAccessInfo>();
|
||||
res->query_privileges_info = std::make_shared<QueryPrivilegesInfo>();
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -1427,7 +1428,7 @@ void Context::checkAccess(const AccessFlags & flags, const StorageID & table_id,
|
||||
void Context::checkAccess(const AccessRightsElement & element) const { checkAccessImpl(element); }
|
||||
void Context::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl(elements); }
|
||||
|
||||
std::shared_ptr<const ContextAccess> Context::getAccess() const
|
||||
std::shared_ptr<const ContextAccessWrapper> Context::getAccess() const
|
||||
{
|
||||
/// A helper function to collect parameters for calculating access rights, called with Context::getLocalSharedLock() acquired.
|
||||
auto get_params = [this]()
|
||||
@ -1444,14 +1445,14 @@ std::shared_ptr<const ContextAccess> Context::getAccess() const
|
||||
{
|
||||
SharedLockGuard lock(mutex);
|
||||
if (access && !need_recalculate_access)
|
||||
return access; /// No need to recalculate access rights.
|
||||
return std::make_shared<const ContextAccessWrapper>(access, shared_from_this()); /// No need to recalculate access rights.
|
||||
|
||||
params.emplace(get_params());
|
||||
|
||||
if (access && (access->getParams() == *params))
|
||||
{
|
||||
need_recalculate_access = false;
|
||||
return access; /// No need to recalculate access rights.
|
||||
return std::make_shared<const ContextAccessWrapper>(access, shared_from_this()); /// No need to recalculate access rights.
|
||||
}
|
||||
}
|
||||
|
||||
@ -1471,7 +1472,7 @@ std::shared_ptr<const ContextAccess> Context::getAccess() const
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
return std::make_shared<const ContextAccessWrapper>(res, shared_from_this());
|
||||
}
|
||||
|
||||
RowPolicyFilterPtr Context::getRowPolicyFilter(const String & database, const String & table_name, RowPolicyFilterType filter_type) const
|
||||
@ -1857,6 +1858,15 @@ void Context::addQueryFactoriesInfo(QueryLogFactories factory_type, const String
|
||||
}
|
||||
}
|
||||
|
||||
void Context::addQueryPrivilegesInfo(const String & privilege, bool granted) const
|
||||
{
|
||||
std::lock_guard lock(query_privileges_info->mutex);
|
||||
if (granted)
|
||||
query_privileges_info->used_privileges.emplace(privilege);
|
||||
else
|
||||
query_privileges_info->missing_privileges.emplace(privilege);
|
||||
}
|
||||
|
||||
static bool findIdentifier(const ASTFunction * function)
|
||||
{
|
||||
if (!function || !function->arguments)
|
||||
@ -2538,6 +2548,7 @@ void Context::makeQueryContext()
|
||||
local_read_query_throttler.reset();
|
||||
local_write_query_throttler.reset();
|
||||
backups_query_throttler.reset();
|
||||
query_privileges_info = std::make_shared<QueryPrivilegesInfo>(*query_privileges_info);
|
||||
}
|
||||
|
||||
void Context::makeQueryContextForMerge(const MergeTreeSettings & merge_tree_settings)
|
||||
|
||||
@ -50,6 +50,7 @@ class ASTSelectQuery;
|
||||
|
||||
struct ContextSharedPart;
|
||||
class ContextAccess;
|
||||
class ContextAccessWrapper;
|
||||
struct User;
|
||||
using UserPtr = std::shared_ptr<const User>;
|
||||
struct SettingsProfilesInfo;
|
||||
@ -403,9 +404,31 @@ public:
|
||||
mutable std::mutex mutex;
|
||||
};
|
||||
|
||||
struct QueryPrivilegesInfo
|
||||
{
|
||||
QueryPrivilegesInfo() = default;
|
||||
|
||||
QueryPrivilegesInfo(const QueryPrivilegesInfo & rhs)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(rhs.mutex);
|
||||
used_privileges = rhs.used_privileges;
|
||||
missing_privileges = rhs.missing_privileges;
|
||||
}
|
||||
|
||||
QueryPrivilegesInfo(QueryPrivilegesInfo && rhs) = delete;
|
||||
|
||||
std::unordered_set<std::string> used_privileges TSA_GUARDED_BY(mutex);
|
||||
std::unordered_set<std::string> missing_privileges TSA_GUARDED_BY(mutex);
|
||||
|
||||
mutable std::mutex mutex;
|
||||
};
|
||||
|
||||
using QueryPrivilegesInfoPtr = std::shared_ptr<QueryPrivilegesInfo>;
|
||||
|
||||
protected:
|
||||
/// Needs to be changed while having const context in factories methods
|
||||
mutable QueryFactoriesInfo query_factories_info;
|
||||
QueryPrivilegesInfoPtr query_privileges_info;
|
||||
/// Query metrics for reading data asynchronously with IAsynchronousReader.
|
||||
mutable std::shared_ptr<AsyncReadCounters> async_read_counters;
|
||||
|
||||
@ -612,7 +635,7 @@ public:
|
||||
void checkAccess(const AccessRightsElement & element) const;
|
||||
void checkAccess(const AccessRightsElements & elements) const;
|
||||
|
||||
std::shared_ptr<const ContextAccess> getAccess() const;
|
||||
std::shared_ptr<const ContextAccessWrapper> getAccess() const;
|
||||
|
||||
RowPolicyFilterPtr getRowPolicyFilter(const String & database, const String & table_name, RowPolicyFilterType filter_type) const;
|
||||
|
||||
@ -741,6 +764,10 @@ public:
|
||||
QueryFactoriesInfo getQueryFactoriesInfo() const;
|
||||
void addQueryFactoriesInfo(QueryLogFactories factory_type, const String & created_object) const;
|
||||
|
||||
const QueryPrivilegesInfo & getQueryPrivilegesInfo() const { return *getQueryPrivilegesInfoPtr(); }
|
||||
QueryPrivilegesInfoPtr getQueryPrivilegesInfoPtr() const { return query_privileges_info; }
|
||||
void addQueryPrivilegesInfo(const String & privilege, bool granted) const;
|
||||
|
||||
/// For table functions s3/file/url/hdfs/input we can use structure from
|
||||
/// insertion table depending on select expression.
|
||||
StoragePtr executeTableFunction(const ASTPtr & table_expression, const ASTSelectQuery * select_query_hint = nullptr);
|
||||
|
||||
@ -129,6 +129,7 @@ public:
|
||||
static constexpr const char * SYSTEM_DATABASE = "system";
|
||||
static constexpr const char * INFORMATION_SCHEMA = "information_schema";
|
||||
static constexpr const char * INFORMATION_SCHEMA_UPPERCASE = "INFORMATION_SCHEMA";
|
||||
static constexpr const char * DEFAULT_DATABASE = "default";
|
||||
|
||||
/// Returns true if a passed name is one of the predefined databases' names.
|
||||
static bool isPredefinedDatabase(std::string_view database_name);
|
||||
|
||||
@ -49,8 +49,9 @@ namespace ErrorCodes
|
||||
|
||||
static std::unordered_set<const ActionsDAG::Node *> processShortCircuitFunctions(const ActionsDAG & actions_dag, ShortCircuitFunctionEvaluation short_circuit_function_evaluation);
|
||||
|
||||
ExpressionActions::ExpressionActions(ActionsDAGPtr actions_dag_, const ExpressionActionsSettings & settings_)
|
||||
: settings(settings_)
|
||||
ExpressionActions::ExpressionActions(ActionsDAGPtr actions_dag_, const ExpressionActionsSettings & settings_, bool project_inputs_)
|
||||
: project_inputs(project_inputs_)
|
||||
, settings(settings_)
|
||||
{
|
||||
actions_dag = actions_dag_->clone();
|
||||
|
||||
@ -757,7 +758,7 @@ void ExpressionActions::execute(Block & block, size_t & num_rows, bool dry_run,
|
||||
}
|
||||
}
|
||||
|
||||
if (actions_dag->isInputProjected())
|
||||
if (project_inputs)
|
||||
{
|
||||
block.clear();
|
||||
}
|
||||
@ -862,7 +863,7 @@ std::string ExpressionActions::dumpActions() const
|
||||
for (const auto & output_column : output_columns)
|
||||
ss << output_column.name << " " << output_column.type->getName() << "\n";
|
||||
|
||||
ss << "\nproject input: " << actions_dag->isInputProjected() << "\noutput positions:";
|
||||
ss << "\noutput positions:";
|
||||
for (auto pos : result_positions)
|
||||
ss << " " << pos;
|
||||
ss << "\n";
|
||||
@ -926,7 +927,6 @@ JSONBuilder::ItemPtr ExpressionActions::toTree() const
|
||||
map->add("Actions", std::move(actions_array));
|
||||
map->add("Outputs", std::move(outputs_array));
|
||||
map->add("Positions", std::move(positions_array));
|
||||
map->add("Project Input", actions_dag->isInputProjected());
|
||||
|
||||
return map;
|
||||
}
|
||||
@ -980,7 +980,7 @@ void ExpressionActionsChain::addStep(NameSet non_constant_inputs)
|
||||
if (column.column && isColumnConst(*column.column) && non_constant_inputs.contains(column.name))
|
||||
column.column = nullptr;
|
||||
|
||||
steps.push_back(std::make_unique<ExpressionActionsStep>(std::make_shared<ActionsDAG>(columns)));
|
||||
steps.push_back(std::make_unique<ExpressionActionsStep>(std::make_shared<ActionsAndProjectInputsFlag>(ActionsDAG(columns), false)));
|
||||
}
|
||||
|
||||
void ExpressionActionsChain::finalize()
|
||||
@ -1129,14 +1129,14 @@ void ExpressionActionsChain::JoinStep::finalize(const NameSet & required_output_
|
||||
std::swap(result_columns, new_result_columns);
|
||||
}
|
||||
|
||||
ActionsDAGPtr & ExpressionActionsChain::Step::actions()
|
||||
ActionsAndProjectInputsFlagPtr & ExpressionActionsChain::Step::actions()
|
||||
{
|
||||
return typeid_cast<ExpressionActionsStep &>(*this).actions_dag;
|
||||
return typeid_cast<ExpressionActionsStep &>(*this).actions_and_flags;
|
||||
}
|
||||
|
||||
const ActionsDAGPtr & ExpressionActionsChain::Step::actions() const
|
||||
const ActionsAndProjectInputsFlagPtr & ExpressionActionsChain::Step::actions() const
|
||||
{
|
||||
return typeid_cast<const ExpressionActionsStep &>(*this).actions_dag;
|
||||
return typeid_cast<const ExpressionActionsStep &>(*this).actions_and_flags;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -79,11 +79,13 @@ private:
|
||||
ColumnNumbers result_positions;
|
||||
Block sample_block;
|
||||
|
||||
bool project_inputs = false;
|
||||
|
||||
ExpressionActionsSettings settings;
|
||||
|
||||
public:
|
||||
ExpressionActions() = delete;
|
||||
explicit ExpressionActions(ActionsDAGPtr actions_dag_, const ExpressionActionsSettings & settings_ = {});
|
||||
explicit ExpressionActions(ActionsDAGPtr actions_dag_, const ExpressionActionsSettings & settings_ = {}, bool project_inputs_ = false);
|
||||
ExpressionActions(const ExpressionActions &) = default;
|
||||
ExpressionActions & operator=(const ExpressionActions &) = default;
|
||||
|
||||
@ -173,48 +175,49 @@ struct ExpressionActionsChain : WithContext
|
||||
/// Remove unused result and update required columns
|
||||
virtual void finalize(const NameSet & required_output_) = 0;
|
||||
/// Add projections to expression
|
||||
virtual void prependProjectInput() const = 0;
|
||||
virtual void prependProjectInput() = 0;
|
||||
virtual std::string dump() const = 0;
|
||||
|
||||
/// Only for ExpressionActionsStep
|
||||
ActionsDAGPtr & actions();
|
||||
const ActionsDAGPtr & actions() const;
|
||||
ActionsAndProjectInputsFlagPtr & actions();
|
||||
const ActionsAndProjectInputsFlagPtr & actions() const;
|
||||
};
|
||||
|
||||
struct ExpressionActionsStep : public Step
|
||||
{
|
||||
ActionsDAGPtr actions_dag;
|
||||
ActionsAndProjectInputsFlagPtr actions_and_flags;
|
||||
bool is_final_projection = false;
|
||||
|
||||
explicit ExpressionActionsStep(ActionsDAGPtr actions_dag_, Names required_output_ = Names())
|
||||
explicit ExpressionActionsStep(ActionsAndProjectInputsFlagPtr actiactions_and_flags_, Names required_output_ = Names())
|
||||
: Step(std::move(required_output_))
|
||||
, actions_dag(std::move(actions_dag_))
|
||||
, actions_and_flags(std::move(actiactions_and_flags_))
|
||||
{
|
||||
}
|
||||
|
||||
NamesAndTypesList getRequiredColumns() const override
|
||||
{
|
||||
return actions_dag->getRequiredColumns();
|
||||
return actions_and_flags->dag.getRequiredColumns();
|
||||
}
|
||||
|
||||
ColumnsWithTypeAndName getResultColumns() const override
|
||||
{
|
||||
return actions_dag->getResultColumns();
|
||||
return actions_and_flags->dag.getResultColumns();
|
||||
}
|
||||
|
||||
void finalize(const NameSet & required_output_) override
|
||||
{
|
||||
if (!actions_dag->isOutputProjected())
|
||||
actions_dag->removeUnusedActions(required_output_);
|
||||
if (!is_final_projection)
|
||||
actions_and_flags->dag.removeUnusedActions(required_output_);
|
||||
}
|
||||
|
||||
void prependProjectInput() const override
|
||||
void prependProjectInput() override
|
||||
{
|
||||
actions_dag->projectInput();
|
||||
actions_and_flags->project_input = true;
|
||||
}
|
||||
|
||||
std::string dump() const override
|
||||
{
|
||||
return actions_dag->dumpDAG();
|
||||
return actions_and_flags->dag.dumpDAG();
|
||||
}
|
||||
};
|
||||
|
||||
@ -229,7 +232,7 @@ struct ExpressionActionsChain : WithContext
|
||||
NamesAndTypesList getRequiredColumns() const override { return required_columns; }
|
||||
ColumnsWithTypeAndName getResultColumns() const override { return result_columns; }
|
||||
void finalize(const NameSet & required_output_) override;
|
||||
void prependProjectInput() const override {} /// TODO: remove unused columns before ARRAY JOIN ?
|
||||
void prependProjectInput() override {} /// TODO: remove unused columns before ARRAY JOIN ?
|
||||
std::string dump() const override { return "ARRAY JOIN"; }
|
||||
};
|
||||
|
||||
@ -245,7 +248,7 @@ struct ExpressionActionsChain : WithContext
|
||||
NamesAndTypesList getRequiredColumns() const override { return required_columns; }
|
||||
ColumnsWithTypeAndName getResultColumns() const override { return result_columns; }
|
||||
void finalize(const NameSet & required_output_) override;
|
||||
void prependProjectInput() const override {} /// TODO: remove unused columns before JOIN ?
|
||||
void prependProjectInput() override {} /// TODO: remove unused columns before JOIN ?
|
||||
std::string dump() const override { return "JOIN"; }
|
||||
};
|
||||
|
||||
@ -263,7 +266,7 @@ struct ExpressionActionsChain : WithContext
|
||||
steps.clear();
|
||||
}
|
||||
|
||||
ActionsDAGPtr getLastActions(bool allow_empty = false)
|
||||
ExpressionActionsStep * getLastExpressionStep(bool allow_empty = false)
|
||||
{
|
||||
if (steps.empty())
|
||||
{
|
||||
@ -272,7 +275,15 @@ struct ExpressionActionsChain : WithContext
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty ExpressionActionsChain");
|
||||
}
|
||||
|
||||
return typeid_cast<ExpressionActionsStep *>(steps.back().get())->actions_dag;
|
||||
return typeid_cast<ExpressionActionsStep *>(steps.back().get());
|
||||
}
|
||||
|
||||
ActionsAndProjectInputsFlagPtr getLastActions(bool allow_empty = false)
|
||||
{
|
||||
if (auto * step = getLastExpressionStep(allow_empty))
|
||||
return step->actions_and_flags;
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
Step & getLastStep()
|
||||
@ -286,10 +297,15 @@ struct ExpressionActionsChain : WithContext
|
||||
Step & lastStep(const NamesAndTypesList & columns)
|
||||
{
|
||||
if (steps.empty())
|
||||
steps.emplace_back(std::make_unique<ExpressionActionsStep>(std::make_shared<ActionsDAG>(columns)));
|
||||
return addStep(columns);
|
||||
return *steps.back();
|
||||
}
|
||||
|
||||
Step & addStep(const NamesAndTypesList & columns)
|
||||
{
|
||||
return *steps.emplace_back(std::make_unique<ExpressionActionsStep>(std::make_shared<ActionsAndProjectInputsFlag>(ActionsDAG(columns), false)));
|
||||
}
|
||||
|
||||
std::string dumpChain() const;
|
||||
};
|
||||
|
||||
|
||||
@ -186,7 +186,7 @@ ExpressionAnalyzer::ExpressionAnalyzer(
|
||||
/// Replaces global subqueries with the generated names of temporary tables that will be sent to remote servers.
|
||||
initGlobalSubqueriesAndExternalTables(do_global, is_explain);
|
||||
|
||||
auto temp_actions = std::make_shared<ActionsDAG>(sourceColumns());
|
||||
ActionsDAG temp_actions(sourceColumns());
|
||||
columns_after_array_join = getColumnsAfterArrayJoin(temp_actions, sourceColumns());
|
||||
columns_after_join = analyzeJoin(temp_actions, columns_after_array_join);
|
||||
/// has_aggregation, aggregation_keys, aggregate_descriptions, aggregated_columns.
|
||||
@ -199,7 +199,7 @@ ExpressionAnalyzer::ExpressionAnalyzer(
|
||||
analyzeAggregation(temp_actions);
|
||||
}
|
||||
|
||||
NamesAndTypesList ExpressionAnalyzer::getColumnsAfterArrayJoin(ActionsDAGPtr & actions, const NamesAndTypesList & src_columns)
|
||||
NamesAndTypesList ExpressionAnalyzer::getColumnsAfterArrayJoin(ActionsDAG & actions, const NamesAndTypesList & src_columns)
|
||||
{
|
||||
const auto * select_query = query->as<ASTSelectQuery>();
|
||||
if (!select_query)
|
||||
@ -213,14 +213,14 @@ NamesAndTypesList ExpressionAnalyzer::getColumnsAfterArrayJoin(ActionsDAGPtr & a
|
||||
getRootActionsNoMakeSet(array_join_expression_list, actions, false);
|
||||
|
||||
auto array_join = addMultipleArrayJoinAction(actions, is_array_join_left);
|
||||
auto sample_columns = actions->getResultColumns();
|
||||
auto sample_columns = actions.getResultColumns();
|
||||
array_join->prepare(sample_columns);
|
||||
actions = std::make_shared<ActionsDAG>(sample_columns);
|
||||
actions = ActionsDAG(sample_columns);
|
||||
|
||||
NamesAndTypesList new_columns_after_array_join;
|
||||
NameSet added_columns;
|
||||
|
||||
for (auto & column : actions->getResultColumns())
|
||||
for (auto & column : actions.getResultColumns())
|
||||
{
|
||||
if (syntax->array_join_result_to_source.contains(column.name))
|
||||
{
|
||||
@ -236,7 +236,7 @@ NamesAndTypesList ExpressionAnalyzer::getColumnsAfterArrayJoin(ActionsDAGPtr & a
|
||||
return new_columns_after_array_join;
|
||||
}
|
||||
|
||||
NamesAndTypesList ExpressionAnalyzer::analyzeJoin(ActionsDAGPtr & actions, const NamesAndTypesList & src_columns)
|
||||
NamesAndTypesList ExpressionAnalyzer::analyzeJoin(ActionsDAG & actions, const NamesAndTypesList & src_columns)
|
||||
{
|
||||
const auto * select_query = query->as<ASTSelectQuery>();
|
||||
if (!select_query)
|
||||
@ -246,9 +246,9 @@ NamesAndTypesList ExpressionAnalyzer::analyzeJoin(ActionsDAGPtr & actions, const
|
||||
if (join)
|
||||
{
|
||||
getRootActionsNoMakeSet(analyzedJoin().leftKeysList(), actions, false);
|
||||
auto sample_columns = actions->getNamesAndTypesList();
|
||||
auto sample_columns = actions.getNamesAndTypesList();
|
||||
syntax->analyzed_join->addJoinedColumnsAndCorrectTypes(sample_columns, true);
|
||||
actions = std::make_shared<ActionsDAG>(sample_columns);
|
||||
actions = ActionsDAG(sample_columns);
|
||||
}
|
||||
|
||||
NamesAndTypesList result_columns = src_columns;
|
||||
@ -256,7 +256,7 @@ NamesAndTypesList ExpressionAnalyzer::analyzeJoin(ActionsDAGPtr & actions, const
|
||||
return result_columns;
|
||||
}
|
||||
|
||||
void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions)
|
||||
void ExpressionAnalyzer::analyzeAggregation(ActionsDAG & temp_actions)
|
||||
{
|
||||
/** Find aggregation keys (aggregation_keys), information about aggregate functions (aggregate_descriptions),
|
||||
* as well as a set of columns obtained after the aggregation, if any,
|
||||
@ -272,7 +272,7 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions)
|
||||
|
||||
if (!has_aggregation)
|
||||
{
|
||||
aggregated_columns = temp_actions->getNamesAndTypesList();
|
||||
aggregated_columns = temp_actions.getNamesAndTypesList();
|
||||
return;
|
||||
}
|
||||
|
||||
@ -321,7 +321,7 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions)
|
||||
|
||||
ssize_t group_size = group_elements_ast.size();
|
||||
const auto & column_name = group_elements_ast[j]->getColumnName();
|
||||
const auto * node = temp_actions->tryFindInOutputs(column_name);
|
||||
const auto * node = temp_actions.tryFindInOutputs(column_name);
|
||||
if (!node)
|
||||
throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, "Unknown identifier (in GROUP BY): {}", column_name);
|
||||
|
||||
@ -375,7 +375,7 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions)
|
||||
getRootActionsNoMakeSet(group_asts[i], temp_actions, false);
|
||||
|
||||
const auto & column_name = group_asts[i]->getColumnName();
|
||||
const auto * node = temp_actions->tryFindInOutputs(column_name);
|
||||
const auto * node = temp_actions.tryFindInOutputs(column_name);
|
||||
if (!node)
|
||||
throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, "Unknown identifier (in GROUP BY): {}", column_name);
|
||||
|
||||
@ -434,7 +434,7 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions)
|
||||
has_const_aggregation_keys = select_query->group_by_with_constant_keys;
|
||||
}
|
||||
else
|
||||
aggregated_columns = temp_actions->getNamesAndTypesList();
|
||||
aggregated_columns = temp_actions.getNamesAndTypesList();
|
||||
|
||||
for (const auto & desc : aggregate_descriptions)
|
||||
aggregated_columns.emplace_back(desc.column_name, desc.function->getResultType());
|
||||
@ -465,7 +465,7 @@ SetPtr ExpressionAnalyzer::isPlainStorageSetInSubquery(const ASTPtr & subquery_o
|
||||
return storage_set->getSet();
|
||||
}
|
||||
|
||||
void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts)
|
||||
void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAG & actions, bool only_consts)
|
||||
{
|
||||
LogAST log;
|
||||
ActionsVisitor::Data visitor_data(
|
||||
@ -485,7 +485,7 @@ void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_
|
||||
actions = visitor_data.getActions();
|
||||
}
|
||||
|
||||
void ExpressionAnalyzer::getRootActionsNoMakeSet(const ASTPtr & ast, ActionsDAGPtr & actions, bool only_consts)
|
||||
void ExpressionAnalyzer::getRootActionsNoMakeSet(const ASTPtr & ast, ActionsDAG & actions, bool only_consts)
|
||||
{
|
||||
LogAST log;
|
||||
ActionsVisitor::Data visitor_data(
|
||||
@ -507,7 +507,7 @@ void ExpressionAnalyzer::getRootActionsNoMakeSet(const ASTPtr & ast, ActionsDAGP
|
||||
|
||||
|
||||
void ExpressionAnalyzer::getRootActionsForHaving(
|
||||
const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts)
|
||||
const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAG & actions, bool only_consts)
|
||||
{
|
||||
LogAST log;
|
||||
ActionsVisitor::Data visitor_data(
|
||||
@ -528,7 +528,7 @@ void ExpressionAnalyzer::getRootActionsForHaving(
|
||||
}
|
||||
|
||||
|
||||
void ExpressionAnalyzer::getRootActionsForWindowFunctions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions)
|
||||
void ExpressionAnalyzer::getRootActionsForWindowFunctions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAG & actions)
|
||||
{
|
||||
LogAST log;
|
||||
ActionsVisitor::Data visitor_data(
|
||||
@ -548,7 +548,7 @@ void ExpressionAnalyzer::getRootActionsForWindowFunctions(const ASTPtr & ast, bo
|
||||
}
|
||||
|
||||
|
||||
void ExpressionAnalyzer::makeAggregateDescriptions(ActionsDAGPtr & actions, AggregateDescriptions & descriptions)
|
||||
void ExpressionAnalyzer::makeAggregateDescriptions(ActionsDAG & actions, AggregateDescriptions & descriptions)
|
||||
{
|
||||
for (const ASTPtr & ast : aggregates())
|
||||
{
|
||||
@ -567,7 +567,7 @@ void ExpressionAnalyzer::makeAggregateDescriptions(ActionsDAGPtr & actions, Aggr
|
||||
for (size_t i = 0; i < arguments.size(); ++i)
|
||||
{
|
||||
const std::string & name = arguments[i]->getColumnName();
|
||||
const auto * dag_node = actions->tryFindInOutputs(name);
|
||||
const auto * dag_node = actions.tryFindInOutputs(name);
|
||||
if (!dag_node)
|
||||
{
|
||||
throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER,
|
||||
@ -659,7 +659,7 @@ void ExpressionAnalyzer::makeWindowDescriptionFromAST(const Context & context_,
|
||||
1 /* nulls_direction */));
|
||||
|
||||
auto actions_dag = std::make_shared<ActionsDAG>(aggregated_columns);
|
||||
getRootActions(column_ast, false, actions_dag);
|
||||
getRootActions(column_ast, false, *actions_dag);
|
||||
desc.partition_by_actions.push_back(std::move(actions_dag));
|
||||
}
|
||||
}
|
||||
@ -680,7 +680,7 @@ void ExpressionAnalyzer::makeWindowDescriptionFromAST(const Context & context_,
|
||||
order_by_element.nulls_direction));
|
||||
|
||||
auto actions_dag = std::make_shared<ActionsDAG>(aggregated_columns);
|
||||
getRootActions(column_ast, false, actions_dag);
|
||||
getRootActions(column_ast, false, *actions_dag);
|
||||
desc.order_by_actions.push_back(std::move(actions_dag));
|
||||
}
|
||||
}
|
||||
@ -720,7 +720,7 @@ void ExpressionAnalyzer::makeWindowDescriptionFromAST(const Context & context_,
|
||||
}
|
||||
}
|
||||
|
||||
void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions)
|
||||
void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAG & actions)
|
||||
{
|
||||
auto current_context = getContext();
|
||||
|
||||
@ -737,13 +737,13 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions)
|
||||
desc, elem.definition.get());
|
||||
|
||||
auto [it, inserted] = window_descriptions.insert(
|
||||
{desc.window_name, desc});
|
||||
{elem.name, std::move(desc)});
|
||||
|
||||
if (!inserted)
|
||||
{
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Window '{}' is defined twice in the WINDOW clause",
|
||||
desc.window_name);
|
||||
elem.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -776,7 +776,7 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions)
|
||||
for (size_t i = 0; i < arguments.size(); ++i)
|
||||
{
|
||||
const std::string & name = arguments[i]->getColumnName();
|
||||
const auto * node = actions->tryFindInOutputs(name);
|
||||
const auto * node = actions.tryFindInOutputs(name);
|
||||
|
||||
if (!node)
|
||||
{
|
||||
@ -817,13 +817,14 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions)
|
||||
{
|
||||
const auto & definition = function_node.window_definition->as<
|
||||
const ASTWindowDefinition &>();
|
||||
auto default_window_name = definition.getDefaultWindowName();
|
||||
WindowDescription desc;
|
||||
desc.window_name = definition.getDefaultWindowName();
|
||||
desc.window_name = default_window_name;
|
||||
makeWindowDescriptionFromAST(*current_context, window_descriptions,
|
||||
desc, &definition);
|
||||
|
||||
auto [it, inserted] = window_descriptions.insert(
|
||||
{desc.window_name, desc});
|
||||
{default_window_name, desc});
|
||||
|
||||
if (!inserted)
|
||||
{
|
||||
@ -871,7 +872,7 @@ const ASTSelectQuery * SelectQueryExpressionAnalyzer::getAggregatingQuery() cons
|
||||
}
|
||||
|
||||
/// "Big" ARRAY JOIN.
|
||||
ArrayJoinActionPtr ExpressionAnalyzer::addMultipleArrayJoinAction(ActionsDAGPtr & actions, bool array_join_is_left) const
|
||||
ArrayJoinActionPtr ExpressionAnalyzer::addMultipleArrayJoinAction(ActionsDAG & actions, bool array_join_is_left) const
|
||||
{
|
||||
NameSet result_columns;
|
||||
for (const auto & result_source : syntax->array_join_result_to_source)
|
||||
@ -879,8 +880,8 @@ ArrayJoinActionPtr ExpressionAnalyzer::addMultipleArrayJoinAction(ActionsDAGPtr
|
||||
/// Assign new names to columns, if needed.
|
||||
if (result_source.first != result_source.second)
|
||||
{
|
||||
const auto & node = actions->findInOutputs(result_source.second);
|
||||
actions->getOutputs().push_back(&actions->addAlias(node, result_source.first));
|
||||
const auto & node = actions.findInOutputs(result_source.second);
|
||||
actions.getOutputs().push_back(&actions.addAlias(node, result_source.first));
|
||||
}
|
||||
|
||||
/// Make ARRAY JOIN (replace arrays with their insides) for the columns in these new names.
|
||||
@ -890,7 +891,7 @@ ArrayJoinActionPtr ExpressionAnalyzer::addMultipleArrayJoinAction(ActionsDAGPtr
|
||||
return std::make_shared<ArrayJoinAction>(result_columns, array_join_is_left, getContext());
|
||||
}
|
||||
|
||||
ArrayJoinActionPtr SelectQueryExpressionAnalyzer::appendArrayJoin(ExpressionActionsChain & chain, ActionsDAGPtr & before_array_join, bool only_types)
|
||||
ArrayJoinActionPtr SelectQueryExpressionAnalyzer::appendArrayJoin(ExpressionActionsChain & chain, ActionsAndProjectInputsFlagPtr & before_array_join, bool only_types)
|
||||
{
|
||||
const auto * select_query = getSelectQuery();
|
||||
|
||||
@ -900,9 +901,9 @@ ArrayJoinActionPtr SelectQueryExpressionAnalyzer::appendArrayJoin(ExpressionActi
|
||||
|
||||
ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns());
|
||||
|
||||
getRootActions(array_join_expression_list, only_types, step.actions());
|
||||
getRootActions(array_join_expression_list, only_types, step.actions()->dag);
|
||||
|
||||
auto array_join = addMultipleArrayJoinAction(step.actions(), is_array_join_left);
|
||||
auto array_join = addMultipleArrayJoinAction(step.actions()->dag, is_array_join_left);
|
||||
before_array_join = chain.getLastActions();
|
||||
|
||||
chain.steps.push_back(std::make_unique<ExpressionActionsChain::ArrayJoinStep>(array_join, step.getResultColumns()));
|
||||
@ -916,20 +917,23 @@ bool SelectQueryExpressionAnalyzer::appendJoinLeftKeys(ExpressionActionsChain &
|
||||
{
|
||||
ExpressionActionsChain::Step & step = chain.lastStep(columns_after_array_join);
|
||||
|
||||
getRootActions(analyzedJoin().leftKeysList(), only_types, step.actions());
|
||||
getRootActions(analyzedJoin().leftKeysList(), only_types, step.actions()->dag);
|
||||
return true;
|
||||
}
|
||||
|
||||
JoinPtr SelectQueryExpressionAnalyzer::appendJoin(
|
||||
ExpressionActionsChain & chain,
|
||||
ActionsDAGPtr & converting_join_columns)
|
||||
ActionsAndProjectInputsFlagPtr & converting_join_columns)
|
||||
{
|
||||
const ColumnsWithTypeAndName & left_sample_columns = chain.getLastStep().getResultColumns();
|
||||
|
||||
JoinPtr join = makeJoin(*syntax->ast_join, left_sample_columns, converting_join_columns);
|
||||
ActionsDAGPtr converting_actions;
|
||||
JoinPtr join = makeJoin(*syntax->ast_join, left_sample_columns, converting_actions);
|
||||
|
||||
if (converting_join_columns)
|
||||
if (converting_actions)
|
||||
{
|
||||
converting_join_columns = std::make_shared<ActionsAndProjectInputsFlag>();
|
||||
converting_join_columns->dag = std::move(*converting_actions);
|
||||
chain.steps.push_back(std::make_unique<ExpressionActionsChain::ExpressionActionsStep>(converting_join_columns));
|
||||
chain.addStep();
|
||||
}
|
||||
@ -1065,7 +1069,7 @@ static std::unique_ptr<QueryPlan> buildJoinedPlan(
|
||||
rename_dag->getOutputs()[pos] = &alias;
|
||||
}
|
||||
}
|
||||
rename_dag->projectInput();
|
||||
rename_dag->appendInputsForUnusedColumns(joined_plan->getCurrentDataStream().header);
|
||||
auto rename_step = std::make_unique<ExpressionStep>(joined_plan->getCurrentDataStream(), std::move(rename_dag));
|
||||
rename_step->setStepDescription("Rename joined columns");
|
||||
joined_plan->addStep(std::move(rename_step));
|
||||
@ -1166,45 +1170,45 @@ JoinPtr SelectQueryExpressionAnalyzer::makeJoin(
|
||||
return join;
|
||||
}
|
||||
|
||||
ActionsDAGPtr SelectQueryExpressionAnalyzer::appendPrewhere(
|
||||
ActionsAndProjectInputsFlagPtr SelectQueryExpressionAnalyzer::appendPrewhere(
|
||||
ExpressionActionsChain & chain, bool only_types)
|
||||
{
|
||||
const auto * select_query = getSelectQuery();
|
||||
if (!select_query->prewhere())
|
||||
return nullptr;
|
||||
return {};
|
||||
|
||||
Names first_action_names;
|
||||
if (!chain.steps.empty())
|
||||
first_action_names = chain.steps.front()->getRequiredColumns().getNames();
|
||||
|
||||
auto & step = chain.lastStep(sourceColumns());
|
||||
getRootActions(select_query->prewhere(), only_types, step.actions());
|
||||
getRootActions(select_query->prewhere(), only_types, step.actions()->dag);
|
||||
String prewhere_column_name = select_query->prewhere()->getColumnName();
|
||||
step.addRequiredOutput(prewhere_column_name);
|
||||
|
||||
const auto & node = step.actions()->findInOutputs(prewhere_column_name);
|
||||
const auto & node = step.actions()->dag.findInOutputs(prewhere_column_name);
|
||||
auto filter_type = node.result_type;
|
||||
if (!filter_type->canBeUsedInBooleanContext())
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER, "Invalid type for filter in PREWHERE: {}",
|
||||
filter_type->getName());
|
||||
|
||||
ActionsDAGPtr prewhere_actions;
|
||||
ActionsAndProjectInputsFlagPtr prewhere_actions;
|
||||
{
|
||||
/// Remove unused source_columns from prewhere actions.
|
||||
auto tmp_actions_dag = std::make_shared<ActionsDAG>(sourceColumns());
|
||||
ActionsDAG tmp_actions_dag(sourceColumns());
|
||||
getRootActions(select_query->prewhere(), only_types, tmp_actions_dag);
|
||||
/// Constants cannot be removed since they can be used in other parts of the query.
|
||||
/// And if they are not used anywhere, except PREWHERE, they will be removed on the next step.
|
||||
tmp_actions_dag->removeUnusedActions(
|
||||
tmp_actions_dag.removeUnusedActions(
|
||||
NameSet{prewhere_column_name},
|
||||
/* allow_remove_inputs= */ true,
|
||||
/* allow_constant_folding= */ false);
|
||||
|
||||
auto required_columns = tmp_actions_dag->getRequiredColumnsNames();
|
||||
auto required_columns = tmp_actions_dag.getRequiredColumnsNames();
|
||||
NameSet required_source_columns(required_columns.begin(), required_columns.end());
|
||||
required_source_columns.insert(first_action_names.begin(), first_action_names.end());
|
||||
|
||||
auto names = step.actions()->getNames();
|
||||
auto names = step.actions()->dag.getNames();
|
||||
NameSet name_set(names.begin(), names.end());
|
||||
|
||||
for (const auto & column : sourceColumns())
|
||||
@ -1213,13 +1217,13 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendPrewhere(
|
||||
|
||||
Names required_output(name_set.begin(), name_set.end());
|
||||
prewhere_actions = chain.getLastActions();
|
||||
prewhere_actions->removeUnusedActions(required_output);
|
||||
prewhere_actions->dag.removeUnusedActions(required_output);
|
||||
}
|
||||
|
||||
{
|
||||
ActionsDAGPtr actions;
|
||||
auto actions = std::make_shared<ActionsAndProjectInputsFlag>();
|
||||
|
||||
auto required_columns = prewhere_actions->getRequiredColumns();
|
||||
auto required_columns = prewhere_actions->dag.getRequiredColumns();
|
||||
NameSet prewhere_input_names;
|
||||
for (const auto & col : required_columns)
|
||||
prewhere_input_names.insert(col.name);
|
||||
@ -1263,11 +1267,11 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendPrewhere(
|
||||
}
|
||||
}
|
||||
|
||||
actions = std::make_shared<ActionsDAG>(std::move(required_columns));
|
||||
actions->dag = ActionsDAG(required_columns);
|
||||
}
|
||||
else
|
||||
{
|
||||
ColumnsWithTypeAndName columns = prewhere_actions->getResultColumns();
|
||||
ColumnsWithTypeAndName columns = prewhere_actions->dag.getResultColumns();
|
||||
|
||||
for (const auto & column : sourceColumns())
|
||||
{
|
||||
@ -1278,7 +1282,7 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendPrewhere(
|
||||
}
|
||||
}
|
||||
|
||||
actions = std::make_shared<ActionsDAG>(std::move(columns));
|
||||
actions->dag = ActionsDAG(columns);
|
||||
}
|
||||
|
||||
chain.steps.emplace_back(
|
||||
@ -1300,12 +1304,12 @@ bool SelectQueryExpressionAnalyzer::appendWhere(ExpressionActionsChain & chain,
|
||||
|
||||
ExpressionActionsChain::Step & step = chain.lastStep(columns_after_join);
|
||||
|
||||
getRootActions(select_query->where(), only_types, step.actions());
|
||||
getRootActions(select_query->where(), only_types, step.actions()->dag);
|
||||
|
||||
auto where_column_name = select_query->where()->getColumnName();
|
||||
step.addRequiredOutput(where_column_name);
|
||||
|
||||
const auto & node = step.actions()->findInOutputs(where_column_name);
|
||||
const auto & node = step.actions()->dag.findInOutputs(where_column_name);
|
||||
auto filter_type = node.result_type;
|
||||
if (!filter_type->canBeUsedInBooleanContext())
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER, "Invalid type for filter in WHERE: {}",
|
||||
@ -1332,7 +1336,7 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain
|
||||
for (const auto & ast_element : ast->children)
|
||||
{
|
||||
step.addRequiredOutput(ast_element->getColumnName());
|
||||
getRootActions(ast_element, only_types, step.actions());
|
||||
getRootActions(ast_element, only_types, step.actions()->dag);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1341,7 +1345,7 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain
|
||||
for (const auto & ast : asts)
|
||||
{
|
||||
step.addRequiredOutput(ast->getColumnName());
|
||||
getRootActions(ast, only_types, step.actions());
|
||||
getRootActions(ast, only_types, step.actions()->dag);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1350,7 +1354,7 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain
|
||||
for (auto & child : asts)
|
||||
{
|
||||
auto actions_dag = std::make_shared<ActionsDAG>(columns_after_join);
|
||||
getRootActions(child, only_types, actions_dag);
|
||||
getRootActions(child, only_types, *actions_dag);
|
||||
group_by_elements_actions.emplace_back(
|
||||
std::make_shared<ExpressionActions>(actions_dag, ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes)));
|
||||
}
|
||||
@ -1387,7 +1391,7 @@ void SelectQueryExpressionAnalyzer::appendAggregateFunctionsArguments(Expression
|
||||
const ASTFunction & node = typeid_cast<const ASTFunction &>(*ast);
|
||||
if (node.arguments)
|
||||
for (auto & argument : node.arguments->children)
|
||||
getRootActions(argument, only_types, step.actions());
|
||||
getRootActions(argument, only_types, step.actions()->dag);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1409,7 +1413,7 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments(
|
||||
// recursively together with (1b) as ASTFunction::window_definition.
|
||||
if (getSelectQuery()->window())
|
||||
{
|
||||
getRootActionsNoMakeSet(getSelectQuery()->window(), step.actions());
|
||||
getRootActionsNoMakeSet(getSelectQuery()->window(), step.actions()->dag);
|
||||
}
|
||||
|
||||
for (const auto & [_, w] : window_descriptions)
|
||||
@ -1420,7 +1424,7 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments(
|
||||
// definitions (1a).
|
||||
// Requiring a constant reference to a shared pointer to non-const AST
|
||||
// doesn't really look sane, but the visitor does indeed require it.
|
||||
getRootActionsNoMakeSet(f.function_node->clone(), step.actions());
|
||||
getRootActionsNoMakeSet(f.function_node->clone(), step.actions()->dag);
|
||||
|
||||
// (2b) Required function argument columns.
|
||||
for (const auto & a : f.function_node->arguments->children)
|
||||
@ -1442,17 +1446,17 @@ void SelectQueryExpressionAnalyzer::appendExpressionsAfterWindowFunctions(Expres
|
||||
ExpressionActionsChain::Step & step = chain.lastStep(columns_after_window);
|
||||
|
||||
for (const auto & expression : syntax->expressions_with_window_function)
|
||||
getRootActionsForWindowFunctions(expression->clone(), true, step.actions());
|
||||
getRootActionsForWindowFunctions(expression->clone(), true, step.actions()->dag);
|
||||
}
|
||||
|
||||
void SelectQueryExpressionAnalyzer::appendGroupByModifiers(ActionsDAGPtr & before_aggregation, ExpressionActionsChain & chain, bool /* only_types */)
|
||||
void SelectQueryExpressionAnalyzer::appendGroupByModifiers(ActionsDAG & before_aggregation, ExpressionActionsChain & chain, bool /* only_types */)
|
||||
{
|
||||
const auto * select_query = getAggregatingQuery();
|
||||
|
||||
if (!select_query->groupBy() || !(select_query->group_by_with_rollup || select_query->group_by_with_cube))
|
||||
return;
|
||||
|
||||
auto source_columns = before_aggregation->getResultColumns();
|
||||
auto source_columns = before_aggregation.getResultColumns();
|
||||
ColumnsWithTypeAndName result_columns;
|
||||
|
||||
for (const auto & source_column : source_columns)
|
||||
@ -1462,9 +1466,11 @@ void SelectQueryExpressionAnalyzer::appendGroupByModifiers(ActionsDAGPtr & befor
|
||||
else
|
||||
result_columns.push_back(source_column);
|
||||
}
|
||||
ExpressionActionsChain::Step & step = chain.lastStep(before_aggregation->getNamesAndTypesList());
|
||||
auto required_output = chain.getLastStep().required_output;
|
||||
ExpressionActionsChain::Step & step = chain.addStep(before_aggregation.getNamesAndTypesList());
|
||||
step.required_output = std::move(required_output);
|
||||
|
||||
step.actions() = ActionsDAG::makeConvertingActions(source_columns, result_columns, ActionsDAG::MatchColumnsMode::Position);
|
||||
step.actions()->dag = std::move(*ActionsDAG::makeConvertingActions(source_columns, result_columns, ActionsDAG::MatchColumnsMode::Position));
|
||||
}
|
||||
|
||||
void SelectQueryExpressionAnalyzer::appendSelectSkipWindowExpressions(ExpressionActionsChain::Step & step, ASTPtr const & node)
|
||||
@ -1495,7 +1501,7 @@ bool SelectQueryExpressionAnalyzer::appendHaving(ExpressionActionsChain & chain,
|
||||
|
||||
ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns);
|
||||
|
||||
getRootActionsForHaving(select_query->having(), only_types, step.actions());
|
||||
getRootActionsForHaving(select_query->having(), only_types, step.actions()->dag);
|
||||
|
||||
step.addRequiredOutput(select_query->having()->getColumnName());
|
||||
|
||||
@ -1508,13 +1514,13 @@ void SelectQueryExpressionAnalyzer::appendSelect(ExpressionActionsChain & chain,
|
||||
|
||||
ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns);
|
||||
|
||||
getRootActions(select_query->select(), only_types, step.actions());
|
||||
getRootActions(select_query->select(), only_types, step.actions()->dag);
|
||||
|
||||
for (const auto & child : select_query->select()->children)
|
||||
appendSelectSkipWindowExpressions(step, child);
|
||||
}
|
||||
|
||||
ActionsDAGPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChain & chain, bool only_types, bool optimize_read_in_order,
|
||||
ActionsAndProjectInputsFlagPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChain & chain, bool only_types, bool optimize_read_in_order,
|
||||
ManyExpressionActions & order_by_elements_actions)
|
||||
{
|
||||
const auto * select_query = getSelectQuery();
|
||||
@ -1538,7 +1544,7 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChai
|
||||
replaceForPositionalArguments(ast->children.at(0), select_query, ASTSelectQuery::Expression::ORDER_BY);
|
||||
}
|
||||
|
||||
getRootActions(select_query->orderBy(), only_types, step.actions());
|
||||
getRootActions(select_query->orderBy(), only_types, step.actions()->dag);
|
||||
|
||||
bool with_fill = false;
|
||||
|
||||
@ -1601,7 +1607,7 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChai
|
||||
for (const auto & child : select_query->orderBy()->children)
|
||||
{
|
||||
auto actions_dag = std::make_shared<ActionsDAG>(columns_after_join);
|
||||
getRootActions(child, only_types, actions_dag);
|
||||
getRootActions(child, only_types, *actions_dag);
|
||||
order_by_elements_actions.emplace_back(
|
||||
std::make_shared<ExpressionActions>(actions_dag, ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes)));
|
||||
}
|
||||
@ -1628,7 +1634,7 @@ bool SelectQueryExpressionAnalyzer::appendLimitBy(ExpressionActionsChain & chain
|
||||
|
||||
ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns);
|
||||
|
||||
getRootActions(select_query->limitBy(), only_types, step.actions());
|
||||
getRootActions(select_query->limitBy(), only_types, step.actions()->dag);
|
||||
|
||||
NameSet existing_column_names;
|
||||
for (const auto & column : aggregated_columns)
|
||||
@ -1657,7 +1663,7 @@ bool SelectQueryExpressionAnalyzer::appendLimitBy(ExpressionActionsChain & chain
|
||||
return true;
|
||||
}
|
||||
|
||||
ActionsDAGPtr SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActionsChain & chain) const
|
||||
ActionsAndProjectInputsFlagPtr SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActionsChain & chain) const
|
||||
{
|
||||
const auto * select_query = getSelectQuery();
|
||||
|
||||
@ -1705,17 +1711,20 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActio
|
||||
}
|
||||
}
|
||||
|
||||
auto actions = chain.getLastActions();
|
||||
actions->project(result_columns);
|
||||
auto * last_step = chain.getLastExpressionStep();
|
||||
auto & actions = last_step->actions_and_flags;
|
||||
actions->dag.project(result_columns);
|
||||
|
||||
if (!required_result_columns.empty())
|
||||
{
|
||||
result_columns.clear();
|
||||
for (const auto & column : required_result_columns)
|
||||
result_columns.emplace_back(column, std::string{});
|
||||
actions->project(result_columns);
|
||||
actions->dag.project(result_columns);
|
||||
}
|
||||
|
||||
actions->project_input = true;
|
||||
last_step->is_final_projection = true;
|
||||
return actions;
|
||||
}
|
||||
|
||||
@ -1723,14 +1732,13 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActio
|
||||
void ExpressionAnalyzer::appendExpression(ExpressionActionsChain & chain, const ASTPtr & expr, bool only_types)
|
||||
{
|
||||
ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns());
|
||||
getRootActions(expr, only_types, step.actions());
|
||||
getRootActions(expr, only_types, step.actions()->dag);
|
||||
step.addRequiredOutput(expr->getColumnName());
|
||||
}
|
||||
|
||||
|
||||
ActionsDAGPtr ExpressionAnalyzer::getActionsDAG(bool add_aliases, bool project_result)
|
||||
ActionsDAGPtr ExpressionAnalyzer::getActionsDAG(bool add_aliases, bool remove_unused_result)
|
||||
{
|
||||
auto actions_dag = std::make_shared<ActionsDAG>(aggregated_columns);
|
||||
ActionsDAG actions_dag(aggregated_columns);
|
||||
NamesWithAliases result_columns;
|
||||
Names result_names;
|
||||
|
||||
@ -1756,13 +1764,15 @@ ActionsDAGPtr ExpressionAnalyzer::getActionsDAG(bool add_aliases, bool project_r
|
||||
|
||||
if (add_aliases)
|
||||
{
|
||||
if (project_result)
|
||||
actions_dag->project(result_columns);
|
||||
if (remove_unused_result)
|
||||
{
|
||||
actions_dag.project(result_columns);
|
||||
}
|
||||
else
|
||||
actions_dag->addAliases(result_columns);
|
||||
actions_dag.addAliases(result_columns);
|
||||
}
|
||||
|
||||
if (!(add_aliases && project_result))
|
||||
if (!(add_aliases && remove_unused_result))
|
||||
{
|
||||
NameSet name_set(result_names.begin(), result_names.end());
|
||||
/// We will not delete the original columns.
|
||||
@ -1775,22 +1785,22 @@ ActionsDAGPtr ExpressionAnalyzer::getActionsDAG(bool add_aliases, bool project_r
|
||||
}
|
||||
}
|
||||
|
||||
actions_dag->removeUnusedActions(name_set);
|
||||
actions_dag.removeUnusedActions(name_set);
|
||||
}
|
||||
|
||||
return actions_dag;
|
||||
return std::make_unique<ActionsDAG>(std::move(actions_dag));
|
||||
}
|
||||
|
||||
ExpressionActionsPtr ExpressionAnalyzer::getActions(bool add_aliases, bool project_result, CompileExpressions compile_expressions)
|
||||
ExpressionActionsPtr ExpressionAnalyzer::getActions(bool add_aliases, bool remove_unused_result, CompileExpressions compile_expressions)
|
||||
{
|
||||
return std::make_shared<ExpressionActions>(
|
||||
getActionsDAG(add_aliases, project_result), ExpressionActionsSettings::fromContext(getContext(), compile_expressions));
|
||||
getActionsDAG(add_aliases, remove_unused_result), ExpressionActionsSettings::fromContext(getContext(), compile_expressions), add_aliases && remove_unused_result);
|
||||
}
|
||||
|
||||
ActionsDAGPtr ExpressionAnalyzer::getConstActionsDAG(const ColumnsWithTypeAndName & constant_inputs)
|
||||
{
|
||||
auto actions = std::make_shared<ActionsDAG>(constant_inputs);
|
||||
getRootActions(query, true /* no_makeset_for_subqueries */, actions, true /* only_consts */);
|
||||
getRootActions(query, true /* no_makeset_for_subqueries */, *actions, true /* only_consts */);
|
||||
return actions;
|
||||
}
|
||||
|
||||
@ -1805,7 +1815,7 @@ std::unique_ptr<QueryPlan> SelectQueryExpressionAnalyzer::getJoinedPlan()
|
||||
return std::move(joined_plan);
|
||||
}
|
||||
|
||||
ActionsDAGPtr SelectQueryExpressionAnalyzer::simpleSelectActions()
|
||||
ActionsAndProjectInputsFlagPtr SelectQueryExpressionAnalyzer::simpleSelectActions()
|
||||
{
|
||||
ExpressionActionsChain new_chain(getContext());
|
||||
appendSelect(new_chain, false);
|
||||
@ -1845,14 +1855,16 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
||||
ssize_t where_step_num = -1;
|
||||
ssize_t having_step_num = -1;
|
||||
|
||||
ActionsAndProjectInputsFlagPtr prewhere_dag_and_flags;
|
||||
|
||||
auto finalize_chain = [&](ExpressionActionsChain & chain) -> ColumnsWithTypeAndName
|
||||
{
|
||||
if (prewhere_step_num >= 0)
|
||||
{
|
||||
ExpressionActionsChain::Step & step = *chain.steps.at(prewhere_step_num);
|
||||
|
||||
auto required_columns_ = prewhere_info->prewhere_actions->getRequiredColumnsNames();
|
||||
NameSet required_source_columns(required_columns_.begin(), required_columns_.end());
|
||||
auto prewhere_required_columns = prewhere_dag_and_flags->dag.getRequiredColumnsNames();
|
||||
NameSet required_source_columns(prewhere_required_columns.begin(), prewhere_required_columns.end());
|
||||
/// Add required columns to required output in order not to remove them after prewhere execution.
|
||||
/// TODO: add sampling and final execution to common chain.
|
||||
for (const auto & column : additional_required_columns_after_prewhere)
|
||||
@ -1864,6 +1876,13 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
||||
|
||||
chain.finalize();
|
||||
|
||||
if (prewhere_dag_and_flags)
|
||||
{
|
||||
auto dag = std::make_shared<ActionsDAG>(std::move(prewhere_dag_and_flags->dag));
|
||||
prewhere_info = std::make_shared<PrewhereInfo>(std::move(dag), query.prewhere()->getColumnName());
|
||||
prewhere_dag_and_flags.reset();
|
||||
}
|
||||
|
||||
finalize(chain, prewhere_step_num, where_step_num, having_step_num, query);
|
||||
|
||||
auto res = chain.getLastStep().getResultColumns();
|
||||
@ -1914,19 +1933,19 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
||||
filter_info->do_remove_column = true;
|
||||
}
|
||||
|
||||
if (auto actions = query_analyzer.appendPrewhere(chain, !first_stage))
|
||||
if (prewhere_dag_and_flags = query_analyzer.appendPrewhere(chain, !first_stage); prewhere_dag_and_flags)
|
||||
{
|
||||
/// Prewhere is always the first one.
|
||||
prewhere_step_num = 0;
|
||||
prewhere_info = std::make_shared<PrewhereInfo>(actions, query.prewhere()->getColumnName());
|
||||
|
||||
if (allowEarlyConstantFolding(*prewhere_info->prewhere_actions, settings))
|
||||
if (allowEarlyConstantFolding(prewhere_dag_and_flags->dag, settings))
|
||||
{
|
||||
Block before_prewhere_sample = source_header;
|
||||
if (sanitizeBlock(before_prewhere_sample))
|
||||
{
|
||||
auto dag = prewhere_dag_and_flags->dag.clone();
|
||||
ExpressionActions(
|
||||
prewhere_info->prewhere_actions,
|
||||
dag,
|
||||
ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_prewhere_sample);
|
||||
auto & column_elem = before_prewhere_sample.getByName(query.prewhere()->getColumnName());
|
||||
/// If the filter column is a constant, record it.
|
||||
@ -1950,7 +1969,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
||||
{
|
||||
where_step_num = chain.steps.size() - 1;
|
||||
before_where = chain.getLastActions();
|
||||
if (allowEarlyConstantFolding(*before_where, settings))
|
||||
if (allowEarlyConstantFolding(before_where->dag, settings))
|
||||
{
|
||||
Block before_where_sample;
|
||||
if (chain.steps.size() > 1)
|
||||
@ -1960,7 +1979,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
||||
if (sanitizeBlock(before_where_sample))
|
||||
{
|
||||
ExpressionActions(
|
||||
before_where,
|
||||
before_where->dag.clone(),
|
||||
ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_where_sample);
|
||||
|
||||
auto & column_elem
|
||||
@ -1986,7 +2005,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
||||
before_aggregation = chain.getLastActions();
|
||||
|
||||
if (settings.group_by_use_nulls)
|
||||
query_analyzer.appendGroupByModifiers(before_aggregation, chain, only_types);
|
||||
query_analyzer.appendGroupByModifiers(before_aggregation->dag, chain, only_types);
|
||||
|
||||
auto columns_before_aggregation = finalize_chain(chain);
|
||||
|
||||
@ -2033,8 +2052,8 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
||||
true);
|
||||
|
||||
auto & step = chain.lastStep(query_analyzer.aggregated_columns);
|
||||
auto & actions = step.actions();
|
||||
actions = ActionsDAG::merge(std::move(*actions), std::move(*converting));
|
||||
auto & actions = step.actions()->dag;
|
||||
actions = std::move(*ActionsDAG::merge(std::move(actions), std::move(*converting)));
|
||||
}
|
||||
}
|
||||
|
||||
@ -2070,13 +2089,13 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
||||
// the main SELECT, similar to what we do for aggregate functions.
|
||||
if (has_window)
|
||||
{
|
||||
query_analyzer.makeWindowDescriptions(chain.getLastActions());
|
||||
query_analyzer.makeWindowDescriptions(chain.getLastActions()->dag);
|
||||
|
||||
query_analyzer.appendWindowFunctionsArguments(chain, only_types || !first_stage);
|
||||
|
||||
// Build a list of output columns of the window step.
|
||||
// 1) We need the columns that are the output of ExpressionActions.
|
||||
for (const auto & x : chain.getLastActions()->getNamesAndTypesList())
|
||||
for (const auto & x : chain.getLastActions()->dag.getNamesAndTypesList())
|
||||
{
|
||||
query_analyzer.columns_after_window.push_back(x);
|
||||
}
|
||||
@ -2113,7 +2132,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
||||
finalize_chain(chain);
|
||||
|
||||
query_analyzer.appendExpressionsAfterWindowFunctions(chain, only_types || !first_stage);
|
||||
for (const auto & x : chain.getLastActions()->getNamesAndTypesList())
|
||||
for (const auto & x : chain.getLastActions()->dag.getNamesAndTypesList())
|
||||
{
|
||||
query_analyzer.columns_after_window.push_back(x);
|
||||
}
|
||||
@ -2173,7 +2192,6 @@ void ExpressionAnalysisResult::finalize(
|
||||
if (prewhere_step_num >= 0)
|
||||
{
|
||||
const ExpressionActionsChain::Step & step = *chain.steps.at(prewhere_step_num);
|
||||
prewhere_info->prewhere_actions->projectInput(false);
|
||||
|
||||
NameSet columns_to_remove;
|
||||
for (const auto & [name, can_remove] : step.required_output)
|
||||
@ -2206,9 +2224,9 @@ void ExpressionAnalysisResult::finalize(
|
||||
void ExpressionAnalysisResult::removeExtraColumns() const
|
||||
{
|
||||
if (hasWhere())
|
||||
before_where->projectInput();
|
||||
before_where->project_input = true;
|
||||
if (hasHaving())
|
||||
before_having->projectInput();
|
||||
before_having->project_input = true;
|
||||
}
|
||||
|
||||
void ExpressionAnalysisResult::checkActions() const
|
||||
@ -2238,7 +2256,7 @@ std::string ExpressionAnalysisResult::dump() const
|
||||
|
||||
if (before_array_join)
|
||||
{
|
||||
ss << "before_array_join " << before_array_join->dumpDAG() << "\n";
|
||||
ss << "before_array_join " << before_array_join->dag.dumpDAG() << "\n";
|
||||
}
|
||||
|
||||
if (array_join)
|
||||
@ -2248,12 +2266,12 @@ std::string ExpressionAnalysisResult::dump() const
|
||||
|
||||
if (before_join)
|
||||
{
|
||||
ss << "before_join " << before_join->dumpDAG() << "\n";
|
||||
ss << "before_join " << before_join->dag.dumpDAG() << "\n";
|
||||
}
|
||||
|
||||
if (before_where)
|
||||
{
|
||||
ss << "before_where " << before_where->dumpDAG() << "\n";
|
||||
ss << "before_where " << before_where->dag.dumpDAG() << "\n";
|
||||
}
|
||||
|
||||
if (prewhere_info)
|
||||
@ -2268,32 +2286,32 @@ std::string ExpressionAnalysisResult::dump() const
|
||||
|
||||
if (before_aggregation)
|
||||
{
|
||||
ss << "before_aggregation " << before_aggregation->dumpDAG() << "\n";
|
||||
ss << "before_aggregation " << before_aggregation->dag.dumpDAG() << "\n";
|
||||
}
|
||||
|
||||
if (before_having)
|
||||
{
|
||||
ss << "before_having " << before_having->dumpDAG() << "\n";
|
||||
ss << "before_having " << before_having->dag.dumpDAG() << "\n";
|
||||
}
|
||||
|
||||
if (before_window)
|
||||
{
|
||||
ss << "before_window " << before_window->dumpDAG() << "\n";
|
||||
ss << "before_window " << before_window->dag.dumpDAG() << "\n";
|
||||
}
|
||||
|
||||
if (before_order_by)
|
||||
{
|
||||
ss << "before_order_by " << before_order_by->dumpDAG() << "\n";
|
||||
ss << "before_order_by " << before_order_by->dag.dumpDAG() << "\n";
|
||||
}
|
||||
|
||||
if (before_limit_by)
|
||||
{
|
||||
ss << "before_limit_by " << before_limit_by->dumpDAG() << "\n";
|
||||
ss << "before_limit_by " << before_limit_by->dag.dumpDAG() << "\n";
|
||||
}
|
||||
|
||||
if (final_projection)
|
||||
{
|
||||
ss << "final_projection " << final_projection->dumpDAG() << "\n";
|
||||
ss << "final_projection " << final_projection->dag.dumpDAG() << "\n";
|
||||
}
|
||||
|
||||
if (!selected_columns.empty())
|
||||
|
||||
@ -115,10 +115,10 @@ public:
|
||||
|
||||
/// If `ast` is not a SELECT query, just gets all the actions to evaluate the expression.
|
||||
/// If add_aliases, only the calculated values in the desired order and add aliases.
|
||||
/// If also project_result, than only aliases remain in the output block.
|
||||
/// If also remove_unused_result, than only aliases remain in the output block.
|
||||
/// Otherwise, only temporary columns will be deleted from the block.
|
||||
ActionsDAGPtr getActionsDAG(bool add_aliases, bool project_result = true);
|
||||
ExpressionActionsPtr getActions(bool add_aliases, bool project_result = true, CompileExpressions compile_expressions = CompileExpressions::no);
|
||||
ActionsDAGPtr getActionsDAG(bool add_aliases, bool remove_unused_result = true);
|
||||
ExpressionActionsPtr getActions(bool add_aliases, bool remove_unused_result = true, CompileExpressions compile_expressions = CompileExpressions::no);
|
||||
|
||||
/// Get actions to evaluate a constant expression. The function adds constants and applies functions that depend only on constants.
|
||||
/// Does not execute subqueries.
|
||||
@ -139,7 +139,7 @@ public:
|
||||
const WindowDescriptions & windowDescriptions() const { return window_descriptions; }
|
||||
|
||||
void makeWindowDescriptionFromAST(const Context & context, const WindowDescriptions & existing_descriptions, WindowDescription & desc, const IAST * ast);
|
||||
void makeWindowDescriptions(ActionsDAGPtr actions);
|
||||
void makeWindowDescriptions(ActionsDAG & actions);
|
||||
|
||||
/** Checks if subquery is not a plain StorageSet.
|
||||
* Because while making set we will read data from StorageSet which is not allowed.
|
||||
@ -172,34 +172,34 @@ protected:
|
||||
/// Find global subqueries in the GLOBAL IN/JOIN sections. Fills in external_tables.
|
||||
void initGlobalSubqueriesAndExternalTables(bool do_global, bool is_explain);
|
||||
|
||||
ArrayJoinActionPtr addMultipleArrayJoinAction(ActionsDAGPtr & actions, bool is_left) const;
|
||||
ArrayJoinActionPtr addMultipleArrayJoinAction(ActionsDAG & actions, bool is_left) const;
|
||||
|
||||
void getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts = false);
|
||||
void getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAG & actions, bool only_consts = false);
|
||||
|
||||
/** Similar to getRootActions but do not make sets when analyzing IN functions. It's used in
|
||||
* analyzeAggregation which happens earlier than analyzing PREWHERE and WHERE. If we did, the
|
||||
* prepared sets would not be applicable for MergeTree index optimization.
|
||||
*/
|
||||
void getRootActionsNoMakeSet(const ASTPtr & ast, ActionsDAGPtr & actions, bool only_consts = false);
|
||||
void getRootActionsNoMakeSet(const ASTPtr & ast, ActionsDAG & actions, bool only_consts = false);
|
||||
|
||||
void getRootActionsForHaving(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts = false);
|
||||
void getRootActionsForHaving(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAG & actions, bool only_consts = false);
|
||||
|
||||
void getRootActionsForWindowFunctions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions);
|
||||
void getRootActionsForWindowFunctions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAG & actions);
|
||||
|
||||
/** Add aggregation keys to aggregation_keys, aggregate functions to aggregate_descriptions,
|
||||
* Create a set of columns aggregated_columns resulting after the aggregation, if any,
|
||||
* or after all the actions that are normally performed before aggregation.
|
||||
* Set has_aggregation = true if there is GROUP BY or at least one aggregate function.
|
||||
*/
|
||||
void analyzeAggregation(ActionsDAGPtr & temp_actions);
|
||||
void makeAggregateDescriptions(ActionsDAGPtr & actions, AggregateDescriptions & descriptions);
|
||||
void analyzeAggregation(ActionsDAG & temp_actions);
|
||||
void makeAggregateDescriptions(ActionsDAG & actions, AggregateDescriptions & descriptions);
|
||||
|
||||
const ASTSelectQuery * getSelectQuery() const;
|
||||
|
||||
bool isRemoteStorage() const;
|
||||
|
||||
NamesAndTypesList getColumnsAfterArrayJoin(ActionsDAGPtr & actions, const NamesAndTypesList & src_columns);
|
||||
NamesAndTypesList analyzeJoin(ActionsDAGPtr & actions, const NamesAndTypesList & src_columns);
|
||||
NamesAndTypesList getColumnsAfterArrayJoin(ActionsDAG & actions, const NamesAndTypesList & src_columns);
|
||||
NamesAndTypesList analyzeJoin(ActionsDAG & actions, const NamesAndTypesList & src_columns);
|
||||
|
||||
AggregationKeysInfo getAggregationKeysInfo() const noexcept
|
||||
{
|
||||
@ -231,20 +231,20 @@ struct ExpressionAnalysisResult
|
||||
|
||||
bool use_grouping_set_key = false;
|
||||
|
||||
ActionsDAGPtr before_array_join;
|
||||
ActionsAndProjectInputsFlagPtr before_array_join;
|
||||
ArrayJoinActionPtr array_join;
|
||||
ActionsDAGPtr before_join;
|
||||
ActionsDAGPtr converting_join_columns;
|
||||
ActionsAndProjectInputsFlagPtr before_join;
|
||||
ActionsAndProjectInputsFlagPtr converting_join_columns;
|
||||
JoinPtr join;
|
||||
ActionsDAGPtr before_where;
|
||||
ActionsDAGPtr before_aggregation;
|
||||
ActionsDAGPtr before_having;
|
||||
ActionsAndProjectInputsFlagPtr before_where;
|
||||
ActionsAndProjectInputsFlagPtr before_aggregation;
|
||||
ActionsAndProjectInputsFlagPtr before_having;
|
||||
String having_column_name;
|
||||
bool remove_having_filter = false;
|
||||
ActionsDAGPtr before_window;
|
||||
ActionsDAGPtr before_order_by;
|
||||
ActionsDAGPtr before_limit_by;
|
||||
ActionsDAGPtr final_projection;
|
||||
ActionsAndProjectInputsFlagPtr before_window;
|
||||
ActionsAndProjectInputsFlagPtr before_order_by;
|
||||
ActionsAndProjectInputsFlagPtr before_limit_by;
|
||||
ActionsAndProjectInputsFlagPtr final_projection;
|
||||
|
||||
/// Columns from the SELECT list, before renaming them to aliases. Used to
|
||||
/// perform SELECT DISTINCT.
|
||||
@ -351,12 +351,12 @@ public:
|
||||
/// Tables that will need to be sent to remote servers for distributed query processing.
|
||||
const TemporaryTablesMapping & getExternalTables() const { return external_tables; }
|
||||
|
||||
ActionsDAGPtr simpleSelectActions();
|
||||
ActionsAndProjectInputsFlagPtr simpleSelectActions();
|
||||
|
||||
/// These appends are public only for tests
|
||||
void appendSelect(ExpressionActionsChain & chain, bool only_types);
|
||||
/// Deletes all columns except mentioned by SELECT, arranges the remaining columns and renames them to aliases.
|
||||
ActionsDAGPtr appendProjectResult(ExpressionActionsChain & chain) const;
|
||||
ActionsAndProjectInputsFlagPtr appendProjectResult(ExpressionActionsChain & chain) const;
|
||||
|
||||
private:
|
||||
StorageMetadataPtr metadata_snapshot;
|
||||
@ -386,13 +386,13 @@ private:
|
||||
*/
|
||||
|
||||
/// Before aggregation:
|
||||
ArrayJoinActionPtr appendArrayJoin(ExpressionActionsChain & chain, ActionsDAGPtr & before_array_join, bool only_types);
|
||||
ArrayJoinActionPtr appendArrayJoin(ExpressionActionsChain & chain, ActionsAndProjectInputsFlagPtr & before_array_join, bool only_types);
|
||||
bool appendJoinLeftKeys(ExpressionActionsChain & chain, bool only_types);
|
||||
JoinPtr appendJoin(ExpressionActionsChain & chain, ActionsDAGPtr & converting_join_columns);
|
||||
JoinPtr appendJoin(ExpressionActionsChain & chain, ActionsAndProjectInputsFlagPtr & converting_join_columns);
|
||||
|
||||
/// remove_filter is set in ExpressionActionsChain::finalize();
|
||||
/// Columns in `additional_required_columns` will not be removed (they can be used for e.g. sampling or FINAL modifier).
|
||||
ActionsDAGPtr appendPrewhere(ExpressionActionsChain & chain, bool only_types);
|
||||
ActionsAndProjectInputsFlagPtr appendPrewhere(ExpressionActionsChain & chain, bool only_types);
|
||||
bool appendWhere(ExpressionActionsChain & chain, bool only_types);
|
||||
bool appendGroupBy(ExpressionActionsChain & chain, bool only_types, bool optimize_aggregation_in_order, ManyExpressionActions &);
|
||||
void appendAggregateFunctionsArguments(ExpressionActionsChain & chain, bool only_types);
|
||||
@ -401,12 +401,12 @@ private:
|
||||
void appendExpressionsAfterWindowFunctions(ExpressionActionsChain & chain, bool only_types);
|
||||
void appendSelectSkipWindowExpressions(ExpressionActionsChain::Step & step, ASTPtr const & node);
|
||||
|
||||
void appendGroupByModifiers(ActionsDAGPtr & before_aggregation, ExpressionActionsChain & chain, bool only_types);
|
||||
void appendGroupByModifiers(ActionsDAG & before_aggregation, ExpressionActionsChain & chain, bool only_types);
|
||||
|
||||
/// After aggregation:
|
||||
bool appendHaving(ExpressionActionsChain & chain, bool only_types);
|
||||
/// appendSelect
|
||||
ActionsDAGPtr appendOrderBy(ExpressionActionsChain & chain, bool only_types, bool optimize_read_in_order, ManyExpressionActions &);
|
||||
ActionsAndProjectInputsFlagPtr appendOrderBy(ExpressionActionsChain & chain, bool only_types, bool optimize_read_in_order, ManyExpressionActions &);
|
||||
bool appendLimitBy(ExpressionActionsChain & chain, bool only_types);
|
||||
/// appendProjectResult
|
||||
};
|
||||
|
||||
@ -88,6 +88,11 @@
|
||||
#include <Interpreters/ReplaceQueryParameterVisitor.h>
|
||||
#include <Parsers/QueryParameterVisitor.h>
|
||||
|
||||
namespace CurrentMetrics
|
||||
{
|
||||
extern const Metric AttachedTable;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -113,6 +118,8 @@ namespace ErrorCodes
|
||||
extern const int UNKNOWN_STORAGE;
|
||||
extern const int SYNTAX_ERROR;
|
||||
extern const int SUPPORT_IS_DISABLED;
|
||||
extern const int TOO_MANY_TABLES;
|
||||
extern const int TOO_MANY_DATABASES;
|
||||
}
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
@ -138,6 +145,31 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
|
||||
throw Exception(ErrorCodes::DATABASE_ALREADY_EXISTS, "Database {} already exists.", database_name);
|
||||
}
|
||||
|
||||
auto db_num_limit = getContext()->getGlobalContext()->getServerSettings().max_database_num_to_throw;
|
||||
if (db_num_limit > 0)
|
||||
{
|
||||
size_t db_count = DatabaseCatalog::instance().getDatabases().size();
|
||||
std::vector<String> system_databases = {
|
||||
DatabaseCatalog::TEMPORARY_DATABASE,
|
||||
DatabaseCatalog::SYSTEM_DATABASE,
|
||||
DatabaseCatalog::INFORMATION_SCHEMA,
|
||||
DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE,
|
||||
DatabaseCatalog::DEFAULT_DATABASE
|
||||
};
|
||||
|
||||
for (const auto & system_database : system_databases)
|
||||
{
|
||||
if (db_count > 0 && DatabaseCatalog::instance().isDatabaseExist(system_database))
|
||||
db_count--;
|
||||
}
|
||||
|
||||
if (db_count >= db_num_limit)
|
||||
throw Exception(ErrorCodes::TOO_MANY_DATABASES,
|
||||
"Too many databases in the Clickhouse. "
|
||||
"The limit (setting 'max_database_num_to_throw') is set to {}, current number of databases is {}",
|
||||
db_num_limit, db_count);
|
||||
}
|
||||
|
||||
/// Will write file with database metadata, if needed.
|
||||
String database_name_escaped = escapeForFileName(database_name);
|
||||
fs::path metadata_path = fs::weakly_canonical(getContext()->getPath());
|
||||
@ -1543,6 +1575,17 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
|
||||
}
|
||||
}
|
||||
|
||||
UInt64 table_num_limit = getContext()->getGlobalContext()->getServerSettings().max_table_num_to_throw;
|
||||
if (table_num_limit > 0 && create.getDatabase() != DatabaseCatalog::SYSTEM_DATABASE)
|
||||
{
|
||||
UInt64 table_count = CurrentMetrics::get(CurrentMetrics::AttachedTable);
|
||||
if (table_count >= table_num_limit)
|
||||
throw Exception(ErrorCodes::TOO_MANY_TABLES,
|
||||
"Too many tables in the Clickhouse. "
|
||||
"The limit (setting 'max_table_num_to_throw') is set to {}, current number of tables is {}",
|
||||
table_num_limit, table_count);
|
||||
}
|
||||
|
||||
database->createTable(getContext(), create.getTable(), res, query_ptr);
|
||||
|
||||
/// Move table data to the proper place. Wo do not move data earlier to avoid situations
|
||||
|
||||
@ -26,7 +26,8 @@
|
||||
#include <Processors/Transforms/CountingTransform.h>
|
||||
#include <Processors/Transforms/ExpressionTransform.h>
|
||||
#include <Processors/Transforms/MaterializingTransform.h>
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
#include <Processors/Transforms/PlanSquashingTransform.h>
|
||||
#include <Processors/Transforms/getSourceFromASTInsertQuery.h>
|
||||
#include <Processors/QueryPlan/QueryPlan.h>
|
||||
#include <QueryPipeline/QueryPipelineBuilder.h>
|
||||
@ -625,9 +626,15 @@ BlockIO InterpreterInsertQuery::execute()
|
||||
{
|
||||
bool table_prefers_large_blocks = table->prefersLargeBlocks();
|
||||
|
||||
pipeline.addTransform(std::make_shared<PlanSquashingTransform>(
|
||||
header,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL,
|
||||
presink_chains.size()));
|
||||
|
||||
pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
|
||||
{
|
||||
return std::make_shared<SimpleSquashingChunksTransform>(
|
||||
return std::make_shared<ApplySquashingTransform>(
|
||||
in_header,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL);
|
||||
@ -683,12 +690,20 @@ BlockIO InterpreterInsertQuery::execute()
|
||||
{
|
||||
bool table_prefers_large_blocks = table->prefersLargeBlocks();
|
||||
|
||||
auto squashing = std::make_shared<SimpleSquashingChunksTransform>(
|
||||
chain.getInputHeader(),
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL);
|
||||
auto squashing = std::make_shared<ApplySquashingTransform>(
|
||||
chain.getInputHeader(),
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL);
|
||||
|
||||
chain.addSource(std::move(squashing));
|
||||
|
||||
auto balancing = std::make_shared<PlanSquashingTransform>(
|
||||
chain.getInputHeader(),
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL,
|
||||
presink_chains.size());
|
||||
|
||||
chain.addSource(std::move(balancing));
|
||||
}
|
||||
|
||||
auto context_ptr = getContext();
|
||||
|
||||
@ -175,11 +175,10 @@ FilterDAGInfoPtr generateFilterActions(
|
||||
/// Using separate expression analyzer to prevent any possible alias injection
|
||||
auto syntax_result = TreeRewriter(context).analyzeSelect(query_ast, TreeRewriterResult({}, storage, storage_snapshot));
|
||||
SelectQueryExpressionAnalyzer analyzer(query_ast, syntax_result, context, metadata_snapshot, {}, false, {}, prepared_sets);
|
||||
filter_info->actions = analyzer.simpleSelectActions();
|
||||
filter_info->actions = std::make_unique<ActionsDAG>(std::move(analyzer.simpleSelectActions()->dag));
|
||||
|
||||
filter_info->column_name = expr_list->children.at(0)->getColumnName();
|
||||
filter_info->actions->removeUnusedActions(NameSet{filter_info->column_name});
|
||||
filter_info->actions->projectInput(false);
|
||||
|
||||
for (const auto * node : filter_info->actions->getInputs())
|
||||
filter_info->actions->getOutputs().push_back(node);
|
||||
@ -1078,15 +1077,15 @@ Block InterpreterSelectQuery::getSampleBlockImpl()
|
||||
// with this code. See
|
||||
// https://github.com/ClickHouse/ClickHouse/issues/19857 for details.
|
||||
if (analysis_result.before_window)
|
||||
return analysis_result.before_window->getResultColumns();
|
||||
return analysis_result.before_window->dag.getResultColumns();
|
||||
|
||||
// NOTE: should not handle before_limit_by specially since
|
||||
// WithMergeableState does not process LIMIT BY
|
||||
|
||||
return analysis_result.before_order_by->getResultColumns();
|
||||
return analysis_result.before_order_by->dag.getResultColumns();
|
||||
}
|
||||
|
||||
Block header = analysis_result.before_aggregation->getResultColumns();
|
||||
Block header = analysis_result.before_aggregation->dag.getResultColumns();
|
||||
|
||||
Block res;
|
||||
|
||||
@ -1124,18 +1123,18 @@ Block InterpreterSelectQuery::getSampleBlockImpl()
|
||||
// It's different from selected_columns, see the comment above for
|
||||
// WithMergeableState stage.
|
||||
if (analysis_result.before_window)
|
||||
return analysis_result.before_window->getResultColumns();
|
||||
return analysis_result.before_window->dag.getResultColumns();
|
||||
|
||||
// In case of query on remote shards executed up to
|
||||
// WithMergeableStateAfterAggregation*, they can process LIMIT BY,
|
||||
// since the initiator will not apply LIMIT BY again.
|
||||
if (analysis_result.before_limit_by)
|
||||
return analysis_result.before_limit_by->getResultColumns();
|
||||
return analysis_result.before_limit_by->dag.getResultColumns();
|
||||
|
||||
return analysis_result.before_order_by->getResultColumns();
|
||||
return analysis_result.before_order_by->dag.getResultColumns();
|
||||
}
|
||||
|
||||
return analysis_result.final_projection->getResultColumns();
|
||||
return analysis_result.final_projection->dag.getResultColumns();
|
||||
}
|
||||
|
||||
|
||||
@ -1474,6 +1473,9 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
|
||||
if (expressions.hasHaving() && query.group_by_with_totals && (query.group_by_with_rollup || query.group_by_with_cube))
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "WITH TOTALS and WITH ROLLUP or CUBE are not supported together in presence of HAVING");
|
||||
|
||||
if (query.qualify())
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "QUALIFY clause is not supported in the old analyzer");
|
||||
|
||||
if (options.only_analyze)
|
||||
{
|
||||
auto read_nothing = std::make_unique<ReadNothingStep>(source_header);
|
||||
@ -1636,12 +1638,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
|
||||
add_filter_step(parallel_replicas_custom_filter_info, "Parallel replica custom key filter");
|
||||
|
||||
if (expressions.before_array_join)
|
||||
{
|
||||
QueryPlanStepPtr before_array_join_step
|
||||
= std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), expressions.before_array_join);
|
||||
before_array_join_step->setStepDescription("Before ARRAY JOIN");
|
||||
query_plan.addStep(std::move(before_array_join_step));
|
||||
}
|
||||
executeExpression(query_plan, expressions.before_array_join, "Before ARRAY JOIN");
|
||||
|
||||
if (expressions.array_join)
|
||||
{
|
||||
@ -1653,23 +1650,11 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
|
||||
}
|
||||
|
||||
if (expressions.before_join)
|
||||
{
|
||||
QueryPlanStepPtr before_join_step = std::make_unique<ExpressionStep>(
|
||||
query_plan.getCurrentDataStream(),
|
||||
expressions.before_join);
|
||||
before_join_step->setStepDescription("Before JOIN");
|
||||
query_plan.addStep(std::move(before_join_step));
|
||||
}
|
||||
executeExpression(query_plan, expressions.before_join, "Before JOIN");
|
||||
|
||||
/// Optional step to convert key columns to common supertype.
|
||||
if (expressions.converting_join_columns)
|
||||
{
|
||||
QueryPlanStepPtr convert_join_step = std::make_unique<ExpressionStep>(
|
||||
query_plan.getCurrentDataStream(),
|
||||
expressions.converting_join_columns);
|
||||
convert_join_step->setStepDescription("Convert JOIN columns");
|
||||
query_plan.addStep(std::move(convert_join_step));
|
||||
}
|
||||
executeExpression(query_plan, expressions.converting_join_columns, "Convert JOIN columns");
|
||||
|
||||
if (expressions.hasJoin())
|
||||
{
|
||||
@ -2113,7 +2098,6 @@ void InterpreterSelectQuery::applyFiltersToPrewhereInAnalysis(ExpressionAnalysis
|
||||
{
|
||||
/// Execute row level filter in prewhere as a part of "move to prewhere" optimization.
|
||||
analysis.prewhere_info = std::make_shared<PrewhereInfo>(analysis.filter_info->actions, analysis.filter_info->column_name);
|
||||
analysis.prewhere_info->prewhere_actions->projectInput(false);
|
||||
analysis.prewhere_info->remove_prewhere_column = analysis.filter_info->do_remove_column;
|
||||
analysis.prewhere_info->need_filter = true;
|
||||
analysis.filter_info = nullptr;
|
||||
@ -2124,7 +2108,6 @@ void InterpreterSelectQuery::applyFiltersToPrewhereInAnalysis(ExpressionAnalysis
|
||||
/// Add row level security actions to prewhere.
|
||||
analysis.prewhere_info->row_level_filter = analysis.filter_info->actions;
|
||||
analysis.prewhere_info->row_level_column_name = analysis.filter_info->column_name;
|
||||
analysis.prewhere_info->row_level_filter->projectInput(false);
|
||||
analysis.filter_info = nullptr;
|
||||
}
|
||||
}
|
||||
@ -2333,7 +2316,7 @@ std::optional<UInt64> InterpreterSelectQuery::getTrivialCount(UInt64 max_paralle
|
||||
}
|
||||
if (analysis_result.hasWhere())
|
||||
{
|
||||
filter_nodes.push_back(&analysis_result.before_where->findInOutputs(analysis_result.where_column_name));
|
||||
filter_nodes.push_back(&analysis_result.before_where->dag.findInOutputs(analysis_result.where_column_name));
|
||||
}
|
||||
|
||||
auto filter_actions_dag = ActionsDAG::buildFilterActionsDAG(filter_nodes);
|
||||
@ -2399,7 +2382,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
|
||||
auto column = ColumnAggregateFunction::create(func);
|
||||
column->insertFrom(place);
|
||||
|
||||
Block header = analysis_result.before_aggregation->getResultColumns();
|
||||
Block header = analysis_result.before_aggregation->dag.getResultColumns();
|
||||
size_t arguments_size = desc.argument_names.size();
|
||||
DataTypes argument_types(arguments_size);
|
||||
for (size_t j = 0; j < arguments_size; ++j)
|
||||
@ -2554,6 +2537,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
|
||||
|
||||
query_info.storage_limits = std::make_shared<StorageLimitsList>(storage_limits);
|
||||
query_info.settings_limit_offset_done = options.settings_limit_offset_done;
|
||||
|
||||
storage->read(query_plan, required_columns, storage_snapshot, query_info, context, processing_stage, max_block_size, max_streams);
|
||||
|
||||
if (context->hasQueryContext() && !options.is_internal)
|
||||
@ -2595,10 +2579,14 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
|
||||
}
|
||||
}
|
||||
|
||||
void InterpreterSelectQuery::executeWhere(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool remove_filter)
|
||||
void InterpreterSelectQuery::executeWhere(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter)
|
||||
{
|
||||
auto dag = expression->dag.clone();
|
||||
if (expression->project_input)
|
||||
dag->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
|
||||
|
||||
auto where_step = std::make_unique<FilterStep>(
|
||||
query_plan.getCurrentDataStream(), expression, getSelectQuery().where()->getColumnName(), remove_filter);
|
||||
query_plan.getCurrentDataStream(), std::move(dag), getSelectQuery().where()->getColumnName(), remove_filter);
|
||||
|
||||
where_step->setStepDescription("WHERE");
|
||||
query_plan.addStep(std::move(where_step));
|
||||
@ -2672,11 +2660,9 @@ static GroupingSetsParamsList getAggregatorGroupingSetsParams(const SelectQueryE
|
||||
return result;
|
||||
}
|
||||
|
||||
void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info)
|
||||
void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info)
|
||||
{
|
||||
auto expression_before_aggregation = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), expression);
|
||||
expression_before_aggregation->setStepDescription("Before GROUP BY");
|
||||
query_plan.addStep(std::move(expression_before_aggregation));
|
||||
executeExpression(query_plan, expression, "Before GROUP BY");
|
||||
|
||||
AggregateDescriptions aggregates = query_analyzer->aggregates();
|
||||
const Settings & settings = context->getSettingsRef();
|
||||
@ -2767,10 +2753,14 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool
|
||||
}
|
||||
|
||||
|
||||
void InterpreterSelectQuery::executeHaving(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool remove_filter)
|
||||
void InterpreterSelectQuery::executeHaving(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter)
|
||||
{
|
||||
auto dag = expression->dag.clone();
|
||||
if (expression->project_input)
|
||||
dag->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
|
||||
|
||||
auto having_step
|
||||
= std::make_unique<FilterStep>(query_plan.getCurrentDataStream(), expression, getSelectQuery().having()->getColumnName(), remove_filter);
|
||||
= std::make_unique<FilterStep>(query_plan.getCurrentDataStream(), std::move(dag), getSelectQuery().having()->getColumnName(), remove_filter);
|
||||
|
||||
having_step->setStepDescription("HAVING");
|
||||
query_plan.addStep(std::move(having_step));
|
||||
@ -2778,15 +2768,23 @@ void InterpreterSelectQuery::executeHaving(QueryPlan & query_plan, const Actions
|
||||
|
||||
|
||||
void InterpreterSelectQuery::executeTotalsAndHaving(
|
||||
QueryPlan & query_plan, bool has_having, const ActionsDAGPtr & expression, bool remove_filter, bool overflow_row, bool final)
|
||||
QueryPlan & query_plan, bool has_having, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter, bool overflow_row, bool final)
|
||||
{
|
||||
ActionsDAGPtr dag;
|
||||
if (expression)
|
||||
{
|
||||
dag = expression->dag.clone();
|
||||
if (expression->project_input)
|
||||
dag->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
|
||||
}
|
||||
|
||||
const Settings & settings = context->getSettingsRef();
|
||||
|
||||
auto totals_having_step = std::make_unique<TotalsHavingStep>(
|
||||
query_plan.getCurrentDataStream(),
|
||||
query_analyzer->aggregates(),
|
||||
overflow_row,
|
||||
expression,
|
||||
std::move(dag),
|
||||
has_having ? getSelectQuery().having()->getColumnName() : "",
|
||||
remove_filter,
|
||||
settings.totals_mode,
|
||||
@ -2819,12 +2817,16 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific
|
||||
query_plan.addStep(std::move(step));
|
||||
}
|
||||
|
||||
void InterpreterSelectQuery::executeExpression(QueryPlan & query_plan, const ActionsDAGPtr & expression, const std::string & description)
|
||||
void InterpreterSelectQuery::executeExpression(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, const std::string & description)
|
||||
{
|
||||
if (!expression)
|
||||
return;
|
||||
|
||||
auto expression_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), expression);
|
||||
auto dag = expression->dag.clone();
|
||||
if (expression->project_input)
|
||||
dag->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
|
||||
|
||||
auto expression_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(dag));
|
||||
|
||||
expression_step->setStepDescription(description);
|
||||
query_plan.addStep(std::move(expression_step));
|
||||
@ -2994,11 +2996,9 @@ void InterpreterSelectQuery::executeMergeSorted(QueryPlan & query_plan, const st
|
||||
}
|
||||
|
||||
|
||||
void InterpreterSelectQuery::executeProjection(QueryPlan & query_plan, const ActionsDAGPtr & expression)
|
||||
void InterpreterSelectQuery::executeProjection(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression)
|
||||
{
|
||||
auto projection_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), expression);
|
||||
projection_step->setStepDescription("Projection");
|
||||
query_plan.addStep(std::move(projection_step));
|
||||
executeExpression(query_plan, expression, "Projection");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -174,13 +174,13 @@ private:
|
||||
|
||||
/// Different stages of query execution.
|
||||
void executeFetchColumns(QueryProcessingStage::Enum processing_stage, QueryPlan & query_plan);
|
||||
void executeWhere(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool remove_filter);
|
||||
void executeWhere(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter);
|
||||
void executeAggregation(
|
||||
QueryPlan & query_plan, const ActionsDAGPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info);
|
||||
QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info);
|
||||
void executeMergeAggregated(QueryPlan & query_plan, bool overflow_row, bool final, bool has_grouping_sets);
|
||||
void executeTotalsAndHaving(QueryPlan & query_plan, bool has_having, const ActionsDAGPtr & expression, bool remove_filter, bool overflow_row, bool final);
|
||||
void executeHaving(QueryPlan & query_plan, const ActionsDAGPtr & expression, bool remove_filter);
|
||||
static void executeExpression(QueryPlan & query_plan, const ActionsDAGPtr & expression, const std::string & description);
|
||||
void executeTotalsAndHaving(QueryPlan & query_plan, bool has_having, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter, bool overflow_row, bool final);
|
||||
void executeHaving(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter);
|
||||
static void executeExpression(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, const std::string & description);
|
||||
/// FIXME should go through ActionsDAG to behave as a proper function
|
||||
void executeWindow(QueryPlan & query_plan);
|
||||
void executeOrder(QueryPlan & query_plan, InputOrderInfoPtr sorting_info);
|
||||
@ -191,7 +191,7 @@ private:
|
||||
void executeLimitBy(QueryPlan & query_plan);
|
||||
void executeLimit(QueryPlan & query_plan);
|
||||
void executeOffset(QueryPlan & query_plan);
|
||||
static void executeProjection(QueryPlan & query_plan, const ActionsDAGPtr & expression);
|
||||
static void executeProjection(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression);
|
||||
void executeDistinct(QueryPlan & query_plan, bool before_order, Names columns, bool pre_distinct);
|
||||
void executeExtremes(QueryPlan & query_plan);
|
||||
void executeSubqueriesInSetsAndJoins(QueryPlan & query_plan);
|
||||
|
||||
@ -1137,9 +1137,9 @@ void MutationsInterpreter::prepareMutationStages(std::vector<Stage> & prepared_s
|
||||
for (const auto & kv : stage.column_to_updated)
|
||||
{
|
||||
auto column_name = kv.second->getColumnName();
|
||||
const auto & dag_node = actions->findInOutputs(column_name);
|
||||
const auto & alias = actions->addAlias(dag_node, kv.first);
|
||||
actions->addOrReplaceInOutputs(alias);
|
||||
const auto & dag_node = actions->dag.findInOutputs(column_name);
|
||||
const auto & alias = actions->dag.addAlias(dag_node, kv.first);
|
||||
actions->dag.addOrReplaceInOutputs(alias);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1202,7 +1202,7 @@ void MutationsInterpreter::Source::read(
|
||||
{
|
||||
ActionsDAG::NodeRawConstPtrs nodes(num_filters);
|
||||
for (size_t i = 0; i < num_filters; ++i)
|
||||
nodes[i] = &steps[i]->actions()->findInOutputs(names[i]);
|
||||
nodes[i] = &steps[i]->actions()->dag.findInOutputs(names[i]);
|
||||
|
||||
filter = ActionsDAG::buildFilterActionsDAG(nodes);
|
||||
}
|
||||
@ -1273,18 +1273,24 @@ QueryPipelineBuilder MutationsInterpreter::addStreamsForLaterStages(const std::v
|
||||
for (size_t i = 0; i < stage.expressions_chain.steps.size(); ++i)
|
||||
{
|
||||
const auto & step = stage.expressions_chain.steps[i];
|
||||
if (step->actions()->hasArrayJoin())
|
||||
if (step->actions()->dag.hasArrayJoin())
|
||||
throw Exception(ErrorCodes::UNEXPECTED_EXPRESSION, "arrayJoin is not allowed in mutations");
|
||||
|
||||
if (i < stage.filter_column_names.size())
|
||||
{
|
||||
auto dag = step->actions()->dag.clone();
|
||||
if (step->actions()->project_input)
|
||||
dag->appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
|
||||
/// Execute DELETEs.
|
||||
plan.addStep(std::make_unique<FilterStep>(plan.getCurrentDataStream(), step->actions(), stage.filter_column_names[i], false));
|
||||
plan.addStep(std::make_unique<FilterStep>(plan.getCurrentDataStream(), dag, stage.filter_column_names[i], false));
|
||||
}
|
||||
else
|
||||
{
|
||||
auto dag = step->actions()->dag.clone();
|
||||
if (step->actions()->project_input)
|
||||
dag->appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
|
||||
/// Execute UPDATE or final projection.
|
||||
plan.addStep(std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), step->actions()));
|
||||
plan.addStep(std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), dag));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -136,6 +136,9 @@ ColumnsDescription QueryLogElement::getColumnsDescription()
|
||||
|
||||
{"used_row_policies", array_low_cardinality_string, "The list of row policies names that were used during query execution."},
|
||||
|
||||
{"used_privileges", array_low_cardinality_string, "Privileges which were successfully checked during query execution."},
|
||||
{"missing_privileges", array_low_cardinality_string, "Privileges that are missing during query execution."},
|
||||
|
||||
{"transaction_id", getTransactionIDDataType(), "The identifier of the transaction in scope of which this query was executed."},
|
||||
|
||||
{"query_cache_usage", std::move(query_cache_usage_datatype), "Usage of the query cache during query execution. Values: 'Unknown' = Status unknown, 'None' = The query result was neither written into nor read from the query cache, 'Write' = The query result was written into the query cache, 'Read' = The query result was read from the query cache."},
|
||||
@ -267,6 +270,8 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const
|
||||
auto & column_storage_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
|
||||
auto & column_table_function_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
|
||||
auto & column_row_policies_names = typeid_cast<ColumnArray &>(*columns[i++]);
|
||||
auto & column_used_privileges = typeid_cast<ColumnArray &>(*columns[i++]);
|
||||
auto & column_missing_privileges = typeid_cast<ColumnArray &>(*columns[i++]);
|
||||
|
||||
auto fill_column = [](const auto & data, ColumnArray & column)
|
||||
{
|
||||
@ -290,6 +295,8 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const
|
||||
fill_column(used_storages, column_storage_factory_objects);
|
||||
fill_column(used_table_functions, column_table_function_factory_objects);
|
||||
fill_column(used_row_policies, column_row_policies_names);
|
||||
fill_column(used_privileges, column_used_privileges);
|
||||
fill_column(missing_privileges, column_missing_privileges);
|
||||
}
|
||||
|
||||
columns[i++]->insert(Tuple{tid.start_csn, tid.local_tid, tid.host_id});
|
||||
|
||||
@ -81,6 +81,8 @@ struct QueryLogElement
|
||||
std::unordered_set<String> used_storages;
|
||||
std::unordered_set<String> used_table_functions;
|
||||
std::set<String> used_row_policies;
|
||||
std::unordered_set<String> used_privileges;
|
||||
std::unordered_set<String> missing_privileges;
|
||||
|
||||
Int32 exception_code{}; // because ErrorCodes are int
|
||||
String exception;
|
||||
|
||||
@ -532,7 +532,7 @@ ContextMutablePtr Session::makeSessionContext()
|
||||
session_context->checkSettingsConstraints(settings_from_auth_server, SettingSource::QUERY);
|
||||
session_context->applySettingsChanges(settings_from_auth_server);
|
||||
|
||||
recordLoginSucess(session_context);
|
||||
recordLoginSuccess(session_context);
|
||||
|
||||
return session_context;
|
||||
}
|
||||
@ -596,7 +596,7 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std:
|
||||
{ session_name_ },
|
||||
max_sessions_for_user);
|
||||
|
||||
recordLoginSucess(session_context);
|
||||
recordLoginSuccess(session_context);
|
||||
|
||||
return session_context;
|
||||
}
|
||||
@ -672,13 +672,13 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t
|
||||
user = query_context->getUser();
|
||||
|
||||
/// Interserver does not create session context
|
||||
recordLoginSucess(query_context);
|
||||
recordLoginSuccess(query_context);
|
||||
|
||||
return query_context;
|
||||
}
|
||||
|
||||
|
||||
void Session::recordLoginSucess(ContextPtr login_context) const
|
||||
void Session::recordLoginSuccess(ContextPtr login_context) const
|
||||
{
|
||||
if (notified_session_log_about_login)
|
||||
return;
|
||||
@ -694,7 +694,7 @@ void Session::recordLoginSucess(ContextPtr login_context) const
|
||||
session_log->addLoginSuccess(auth_id,
|
||||
named_session ? named_session->key.second : "",
|
||||
settings,
|
||||
access,
|
||||
access->getAccess(),
|
||||
getClientInfo(),
|
||||
user);
|
||||
}
|
||||
|
||||
@ -102,8 +102,7 @@ public:
|
||||
private:
|
||||
std::shared_ptr<SessionLog> getSessionLog() const;
|
||||
ContextMutablePtr makeQueryContextImpl(const ClientInfo * client_info_to_copy, ClientInfo * client_info_to_move) const;
|
||||
void recordLoginSucess(ContextPtr login_context) const;
|
||||
|
||||
void recordLoginSuccess(ContextPtr login_context) const;
|
||||
|
||||
mutable bool notified_session_log_about_login = false;
|
||||
const UUID auth_id;
|
||||
|
||||
@ -214,7 +214,7 @@ void SessionLog::addLoginSuccess(const UUID & auth_id,
|
||||
const ClientInfo & client_info,
|
||||
const UserPtr & login_user)
|
||||
{
|
||||
DB::SessionLogElement log_entry(auth_id, SESSION_LOGIN_SUCCESS);
|
||||
SessionLogElement log_entry(auth_id, SESSION_LOGIN_SUCCESS);
|
||||
log_entry.client_info = client_info;
|
||||
|
||||
if (login_user)
|
||||
|
||||
159
src/Interpreters/Squashing.cpp
Normal file
159
src/Interpreters/Squashing.cpp
Normal file
@ -0,0 +1,159 @@
|
||||
#include <vector>
|
||||
#include <Interpreters/Squashing.h>
|
||||
#include <Common/CurrentThread.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
Squashing::Squashing(Block header_, size_t min_block_size_rows_, size_t min_block_size_bytes_)
|
||||
: header(header_)
|
||||
, min_block_size_rows(min_block_size_rows_)
|
||||
, min_block_size_bytes(min_block_size_bytes_)
|
||||
{
|
||||
}
|
||||
|
||||
Chunk Squashing::flush()
|
||||
{
|
||||
return convertToChunk(std::move(chunks_to_merge_vec));
|
||||
}
|
||||
|
||||
Chunk Squashing::squash(Chunk && input_chunk)
|
||||
{
|
||||
if (!input_chunk.hasChunkInfo())
|
||||
return Chunk();
|
||||
|
||||
const auto *info = getInfoFromChunk(input_chunk);
|
||||
return squash(info->chunks);
|
||||
}
|
||||
|
||||
Chunk Squashing::add(Chunk && input_chunk)
|
||||
{
|
||||
if (!input_chunk)
|
||||
return {};
|
||||
|
||||
/// Just read block is already enough.
|
||||
if (isEnoughSize(input_chunk.getNumRows(), input_chunk.bytes()))
|
||||
{
|
||||
/// If no accumulated data, return just read block.
|
||||
if (chunks_to_merge_vec.empty())
|
||||
{
|
||||
chunks_to_merge_vec.push_back(std::move(input_chunk));
|
||||
Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
|
||||
chunks_to_merge_vec.clear();
|
||||
return res_chunk;
|
||||
}
|
||||
|
||||
/// Return accumulated data (maybe it has small size) and place new block to accumulated data.
|
||||
Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
|
||||
chunks_to_merge_vec.clear();
|
||||
changeCurrentSize(input_chunk.getNumRows(), input_chunk.bytes());
|
||||
chunks_to_merge_vec.push_back(std::move(input_chunk));
|
||||
return res_chunk;
|
||||
}
|
||||
|
||||
/// Accumulated block is already enough.
|
||||
if (isEnoughSize(accumulated_size.rows, accumulated_size.bytes))
|
||||
{
|
||||
/// Return accumulated data and place new block to accumulated data.
|
||||
Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
|
||||
chunks_to_merge_vec.clear();
|
||||
changeCurrentSize(input_chunk.getNumRows(), input_chunk.bytes());
|
||||
chunks_to_merge_vec.push_back(std::move(input_chunk));
|
||||
return res_chunk;
|
||||
}
|
||||
|
||||
/// Pushing data into accumulating vector
|
||||
expandCurrentSize(input_chunk.getNumRows(), input_chunk.bytes());
|
||||
chunks_to_merge_vec.push_back(std::move(input_chunk));
|
||||
|
||||
/// If accumulated data is big enough, we send it
|
||||
if (isEnoughSize(accumulated_size.rows, accumulated_size.bytes))
|
||||
{
|
||||
Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
|
||||
changeCurrentSize(0, 0);
|
||||
chunks_to_merge_vec.clear();
|
||||
return res_chunk;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
Chunk Squashing::convertToChunk(std::vector<Chunk> && chunks) const
|
||||
{
|
||||
if (chunks.empty())
|
||||
return {};
|
||||
|
||||
auto info = std::make_shared<ChunksToSquash>();
|
||||
info->chunks = std::move(chunks);
|
||||
|
||||
chunks.clear();
|
||||
|
||||
return Chunk(header.cloneEmptyColumns(), 0, info);
|
||||
}
|
||||
|
||||
Chunk Squashing::squash(std::vector<Chunk> & input_chunks)
|
||||
{
|
||||
Chunk accumulated_chunk;
|
||||
std::vector<IColumn::MutablePtr> mutable_columns = {};
|
||||
size_t rows = 0;
|
||||
for (const Chunk & chunk : input_chunks)
|
||||
rows += chunk.getNumRows();
|
||||
|
||||
{
|
||||
auto & first_chunk = input_chunks[0];
|
||||
Columns columns = first_chunk.detachColumns();
|
||||
for (auto & column : columns)
|
||||
{
|
||||
mutable_columns.push_back(IColumn::mutate(std::move(column)));
|
||||
mutable_columns.back()->reserve(rows);
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 1; i < input_chunks.size(); ++i) // We've already processed the first chunk above
|
||||
{
|
||||
Columns columns = input_chunks[i].detachColumns();
|
||||
for (size_t j = 0, size = mutable_columns.size(); j < size; ++j)
|
||||
{
|
||||
const auto source_column = columns[j];
|
||||
|
||||
mutable_columns[j]->insertRangeFrom(*source_column, 0, source_column->size());
|
||||
}
|
||||
}
|
||||
accumulated_chunk.setColumns(std::move(mutable_columns), rows);
|
||||
return accumulated_chunk;
|
||||
}
|
||||
|
||||
const ChunksToSquash* Squashing::getInfoFromChunk(const Chunk & chunk)
|
||||
{
|
||||
const auto& info = chunk.getChunkInfo();
|
||||
const auto * agg_info = typeid_cast<const ChunksToSquash *>(info.get());
|
||||
|
||||
if (!agg_info)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no ChunksToSquash in ChunkInfoPtr");
|
||||
|
||||
return agg_info;
|
||||
}
|
||||
|
||||
void Squashing::expandCurrentSize(size_t rows, size_t bytes)
|
||||
{
|
||||
accumulated_size.rows += rows;
|
||||
accumulated_size.bytes += bytes;
|
||||
}
|
||||
|
||||
void Squashing::changeCurrentSize(size_t rows, size_t bytes)
|
||||
{
|
||||
accumulated_size.rows = rows;
|
||||
accumulated_size.bytes = bytes;
|
||||
}
|
||||
|
||||
bool Squashing::isEnoughSize(size_t rows, size_t bytes) const
|
||||
{
|
||||
return (!min_block_size_rows && !min_block_size_bytes)
|
||||
|| (min_block_size_rows && rows >= min_block_size_rows)
|
||||
|| (min_block_size_bytes && bytes >= min_block_size_bytes);
|
||||
}
|
||||
}
|
||||
69
src/Interpreters/Squashing.h
Normal file
69
src/Interpreters/Squashing.h
Normal file
@ -0,0 +1,69 @@
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <Core/Block.h>
|
||||
#include <Processors/Chunk.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
struct ChunksToSquash : public ChunkInfo
|
||||
{
|
||||
mutable std::vector<Chunk> chunks = {};
|
||||
};
|
||||
|
||||
/** Merging consecutive passed blocks to specified minimum size.
|
||||
*
|
||||
* (But if one of input blocks has already at least specified size,
|
||||
* then don't merge it with neighbours, even if neighbours are small.)
|
||||
*
|
||||
* Used to prepare blocks to adequate size for INSERT queries,
|
||||
* because such storages as Memory, StripeLog, Log, TinyLog...
|
||||
* store or compress data in blocks exactly as passed to it,
|
||||
* and blocks of small size are not efficient.
|
||||
*
|
||||
* Order of data is kept.
|
||||
*/
|
||||
|
||||
class Squashing
|
||||
{
|
||||
public:
|
||||
explicit Squashing(Block header_, size_t min_block_size_rows_, size_t min_block_size_bytes_);
|
||||
Squashing(Squashing && other) = default;
|
||||
|
||||
Chunk add(Chunk && input_chunk);
|
||||
static Chunk squash(Chunk && input_chunk);
|
||||
Chunk flush();
|
||||
|
||||
bool isDataLeft()
|
||||
{
|
||||
return !chunks_to_merge_vec.empty();
|
||||
}
|
||||
|
||||
Block header;
|
||||
private:
|
||||
struct CurrentSize
|
||||
{
|
||||
size_t rows = 0;
|
||||
size_t bytes = 0;
|
||||
};
|
||||
|
||||
std::vector<Chunk> chunks_to_merge_vec = {};
|
||||
size_t min_block_size_rows;
|
||||
size_t min_block_size_bytes;
|
||||
|
||||
CurrentSize accumulated_size;
|
||||
|
||||
static const ChunksToSquash * getInfoFromChunk(const Chunk & chunk);
|
||||
|
||||
static Chunk squash(std::vector<Chunk> & input_chunks);
|
||||
|
||||
void expandCurrentSize(size_t rows, size_t bytes);
|
||||
void changeCurrentSize(size_t rows, size_t bytes);
|
||||
bool isEnoughSize(size_t rows, size_t bytes) const;
|
||||
|
||||
Chunk convertToChunk(std::vector<Chunk> && chunks) const;
|
||||
};
|
||||
|
||||
}
|
||||
@ -1,145 +0,0 @@
|
||||
#include <Interpreters/SquashingTransform.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
SquashingTransform::SquashingTransform(size_t min_block_size_rows_, size_t min_block_size_bytes_)
|
||||
: min_block_size_rows(min_block_size_rows_)
|
||||
, min_block_size_bytes(min_block_size_bytes_)
|
||||
{
|
||||
}
|
||||
|
||||
Block SquashingTransform::add(Block && input_block)
|
||||
{
|
||||
return addImpl<Block &&>(std::move(input_block));
|
||||
}
|
||||
|
||||
Block SquashingTransform::add(const Block & input_block)
|
||||
{
|
||||
return addImpl<const Block &>(input_block);
|
||||
}
|
||||
|
||||
/*
|
||||
* To minimize copying, accept two types of argument: const reference for output
|
||||
* stream, and rvalue reference for input stream, and decide whether to copy
|
||||
* inside this function. This allows us not to copy Block unless we absolutely
|
||||
* have to.
|
||||
*/
|
||||
template <typename ReferenceType>
|
||||
Block SquashingTransform::addImpl(ReferenceType input_block)
|
||||
{
|
||||
/// End of input stream.
|
||||
if (!input_block)
|
||||
{
|
||||
Block to_return;
|
||||
std::swap(to_return, accumulated_block);
|
||||
return to_return;
|
||||
}
|
||||
|
||||
/// Just read block is already enough.
|
||||
if (isEnoughSize(input_block))
|
||||
{
|
||||
/// If no accumulated data, return just read block.
|
||||
if (!accumulated_block)
|
||||
{
|
||||
return std::move(input_block);
|
||||
}
|
||||
|
||||
/// Return accumulated data (maybe it has small size) and place new block to accumulated data.
|
||||
Block to_return = std::move(input_block);
|
||||
std::swap(to_return, accumulated_block);
|
||||
return to_return;
|
||||
}
|
||||
|
||||
/// Accumulated block is already enough.
|
||||
if (isEnoughSize(accumulated_block))
|
||||
{
|
||||
/// Return accumulated data and place new block to accumulated data.
|
||||
Block to_return = std::move(input_block);
|
||||
std::swap(to_return, accumulated_block);
|
||||
return to_return;
|
||||
}
|
||||
|
||||
append<ReferenceType>(std::move(input_block));
|
||||
if (isEnoughSize(accumulated_block))
|
||||
{
|
||||
Block to_return;
|
||||
std::swap(to_return, accumulated_block);
|
||||
return to_return;
|
||||
}
|
||||
|
||||
/// Squashed block is not ready.
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
template <typename ReferenceType>
|
||||
void SquashingTransform::append(ReferenceType input_block)
|
||||
{
|
||||
if (!accumulated_block)
|
||||
{
|
||||
accumulated_block = std::move(input_block);
|
||||
return;
|
||||
}
|
||||
|
||||
assert(blocksHaveEqualStructure(input_block, accumulated_block));
|
||||
|
||||
try
|
||||
{
|
||||
for (size_t i = 0, size = accumulated_block.columns(); i < size; ++i)
|
||||
{
|
||||
const auto source_column = input_block.getByPosition(i).column;
|
||||
|
||||
auto mutable_column = IColumn::mutate(std::move(accumulated_block.getByPosition(i).column));
|
||||
mutable_column->insertRangeFrom(*source_column, 0, source_column->size());
|
||||
accumulated_block.getByPosition(i).column = std::move(mutable_column);
|
||||
}
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
/// add() may be called again even after a previous add() threw an exception.
|
||||
/// Keep accumulated_block in a valid state.
|
||||
/// Seems ok to discard accumulated data because we're throwing an exception, which the caller will
|
||||
/// hopefully interpret to mean "this block and all *previous* blocks are potentially lost".
|
||||
accumulated_block.clear();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
bool SquashingTransform::isEnoughSize(const Block & block)
|
||||
{
|
||||
size_t rows = 0;
|
||||
size_t bytes = 0;
|
||||
|
||||
for (const auto & [column, type, name] : block)
|
||||
{
|
||||
if (!column)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid column in block.");
|
||||
|
||||
if (!rows)
|
||||
rows = column->size();
|
||||
else if (rows != column->size())
|
||||
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Sizes of columns doesn't match");
|
||||
|
||||
bytes += column->byteSize();
|
||||
}
|
||||
|
||||
return isEnoughSize(rows, bytes);
|
||||
}
|
||||
|
||||
|
||||
bool SquashingTransform::isEnoughSize(size_t rows, size_t bytes) const
|
||||
{
|
||||
return (!min_block_size_rows && !min_block_size_bytes)
|
||||
|| (min_block_size_rows && rows >= min_block_size_rows)
|
||||
|| (min_block_size_bytes && bytes >= min_block_size_bytes);
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,50 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Block.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
/** Merging consecutive passed blocks to specified minimum size.
|
||||
*
|
||||
* (But if one of input blocks has already at least specified size,
|
||||
* then don't merge it with neighbours, even if neighbours are small.)
|
||||
*
|
||||
* Used to prepare blocks to adequate size for INSERT queries,
|
||||
* because such storages as Memory, StripeLog, Log, TinyLog...
|
||||
* store or compress data in blocks exactly as passed to it,
|
||||
* and blocks of small size are not efficient.
|
||||
*
|
||||
* Order of data is kept.
|
||||
*/
|
||||
class SquashingTransform
|
||||
{
|
||||
public:
|
||||
/// Conditions on rows and bytes are OR-ed. If one of them is zero, then corresponding condition is ignored.
|
||||
SquashingTransform(size_t min_block_size_rows_, size_t min_block_size_bytes_);
|
||||
|
||||
/** Add next block and possibly returns squashed block.
|
||||
* At end, you need to pass empty block. As the result for last (empty) block, you will get last Result with ready = true.
|
||||
*/
|
||||
Block add(Block && block);
|
||||
Block add(const Block & block);
|
||||
|
||||
private:
|
||||
size_t min_block_size_rows;
|
||||
size_t min_block_size_bytes;
|
||||
|
||||
Block accumulated_block;
|
||||
|
||||
template <typename ReferenceType>
|
||||
Block addImpl(ReferenceType block);
|
||||
|
||||
template <typename ReferenceType>
|
||||
void append(ReferenceType block);
|
||||
|
||||
bool isEnoughSize(const Block & block);
|
||||
bool isEnoughSize(size_t rows, size_t bytes) const;
|
||||
};
|
||||
|
||||
}
|
||||
@ -44,6 +44,7 @@
|
||||
#include <Formats/FormatFactory.h>
|
||||
#include <Storages/StorageInput.h>
|
||||
|
||||
#include <Access/ContextAccess.h>
|
||||
#include <Access/EnabledQuota.h>
|
||||
#include <Interpreters/ApplyWithGlobalVisitor.h>
|
||||
#include <Interpreters/Context.h>
|
||||
@ -221,6 +222,17 @@ static void logException(ContextPtr context, QueryLogElement & elem, bool log_er
|
||||
LOG_INFO(getLogger("executeQuery"), message);
|
||||
}
|
||||
|
||||
static void
|
||||
addPrivilegesInfoToQueryLogElement(QueryLogElement & element, const ContextPtr context_ptr)
|
||||
{
|
||||
const auto & privileges_info = context_ptr->getQueryPrivilegesInfo();
|
||||
{
|
||||
std::lock_guard lock(privileges_info.mutex);
|
||||
element.used_privileges = privileges_info.used_privileges;
|
||||
element.missing_privileges = privileges_info.missing_privileges;
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
addStatusInfoToQueryLogElement(QueryLogElement & element, const QueryStatusInfo & info, const ASTPtr query_ast, const ContextPtr context_ptr)
|
||||
{
|
||||
@ -286,6 +298,7 @@ addStatusInfoToQueryLogElement(QueryLogElement & element, const QueryStatusInfo
|
||||
}
|
||||
|
||||
element.async_read_counters = context_ptr->getAsyncReadCounters();
|
||||
addPrivilegesInfoToQueryLogElement(element, context_ptr);
|
||||
}
|
||||
|
||||
|
||||
@ -601,6 +614,8 @@ void logExceptionBeforeStart(
|
||||
elem.formatted_query = queryToString(ast);
|
||||
}
|
||||
|
||||
addPrivilegesInfoToQueryLogElement(elem, context);
|
||||
|
||||
// We don't calculate databases, tables and columns when the query isn't able to start
|
||||
|
||||
elem.exception_code = getCurrentExceptionCode();
|
||||
|
||||
@ -31,7 +31,7 @@ TEST(ActionsVisitor, VisitLiteral)
|
||||
size_limits_for_set,
|
||||
size_t(0),
|
||||
name_and_types,
|
||||
std::make_shared<ActionsDAG>(name_and_types),
|
||||
ActionsDAG(name_and_types),
|
||||
std::make_shared<PreparedSets>(),
|
||||
false /* no_subqueries */,
|
||||
false /* no_makeset */,
|
||||
@ -39,7 +39,7 @@ TEST(ActionsVisitor, VisitLiteral)
|
||||
info);
|
||||
ActionsVisitor(visitor_data).visit(ast);
|
||||
auto actions = visitor_data.getActions();
|
||||
ASSERT_EQ(actions->getResultColumns().back().type->getTypeId(), expect_type->getTypeId());
|
||||
ASSERT_EQ(actions.getResultColumns().back().type->getTypeId(), expect_type->getTypeId());
|
||||
}
|
||||
|
||||
TEST(ActionsVisitor, VisitLiteralWithType)
|
||||
@ -61,7 +61,7 @@ TEST(ActionsVisitor, VisitLiteralWithType)
|
||||
size_limits_for_set,
|
||||
size_t(0),
|
||||
name_and_types,
|
||||
std::make_shared<ActionsDAG>(name_and_types),
|
||||
ActionsDAG(name_and_types),
|
||||
std::make_shared<PreparedSets>(),
|
||||
false /* no_subqueries */,
|
||||
false /* no_makeset */,
|
||||
@ -69,5 +69,5 @@ TEST(ActionsVisitor, VisitLiteralWithType)
|
||||
info);
|
||||
ActionsVisitor(visitor_data).visit(ast);
|
||||
auto actions = visitor_data.getActions();
|
||||
ASSERT_EQ(actions->getResultColumns().back().type->getTypeId(), date_type->getTypeId());
|
||||
ASSERT_EQ(actions.getResultColumns().back().type->getTypeId(), date_type->getTypeId());
|
||||
}
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
ActionsChainStep::ActionsChainStep(ActionsDAGPtr actions_,
|
||||
ActionsChainStep::ActionsChainStep(ActionsAndProjectInputsFlagPtr actions_,
|
||||
bool use_actions_nodes_as_output_columns_,
|
||||
ColumnsWithTypeAndName additional_output_columns_)
|
||||
: actions(std::move(actions_))
|
||||
@ -28,12 +28,12 @@ void ActionsChainStep::finalizeInputAndOutputColumns(const NameSet & child_input
|
||||
auto child_input_columns_copy = child_input_columns;
|
||||
|
||||
std::unordered_set<std::string_view> output_nodes_names;
|
||||
output_nodes_names.reserve(actions->getOutputs().size());
|
||||
output_nodes_names.reserve(actions->dag.getOutputs().size());
|
||||
|
||||
for (auto & output_node : actions->getOutputs())
|
||||
for (auto & output_node : actions->dag.getOutputs())
|
||||
output_nodes_names.insert(output_node->result_name);
|
||||
|
||||
for (const auto & node : actions->getNodes())
|
||||
for (const auto & node : actions->dag.getNodes())
|
||||
{
|
||||
auto it = child_input_columns_copy.find(node.result_name);
|
||||
if (it == child_input_columns_copy.end())
|
||||
@ -45,20 +45,20 @@ void ActionsChainStep::finalizeInputAndOutputColumns(const NameSet & child_input
|
||||
if (output_nodes_names.contains(node.result_name))
|
||||
continue;
|
||||
|
||||
actions->getOutputs().push_back(&node);
|
||||
actions->dag.getOutputs().push_back(&node);
|
||||
output_nodes_names.insert(node.result_name);
|
||||
}
|
||||
|
||||
actions->removeUnusedActions();
|
||||
actions->dag.removeUnusedActions();
|
||||
/// TODO: Analyzer fix ActionsDAG input and constant nodes with same name
|
||||
actions->projectInput();
|
||||
actions->project_input = true;
|
||||
initialize();
|
||||
}
|
||||
|
||||
void ActionsChainStep::dump(WriteBuffer & buffer) const
|
||||
{
|
||||
buffer << "DAG" << '\n';
|
||||
buffer << actions->dumpDAG();
|
||||
buffer << actions->dag.dumpDAG();
|
||||
|
||||
if (!available_output_columns.empty())
|
||||
{
|
||||
@ -84,7 +84,7 @@ String ActionsChainStep::dump() const
|
||||
|
||||
void ActionsChainStep::initialize()
|
||||
{
|
||||
auto required_columns_names = actions->getRequiredColumnsNames();
|
||||
auto required_columns_names = actions->dag.getRequiredColumnsNames();
|
||||
input_columns_names = NameSet(required_columns_names.begin(), required_columns_names.end());
|
||||
|
||||
available_output_columns.clear();
|
||||
@ -93,7 +93,7 @@ void ActionsChainStep::initialize()
|
||||
{
|
||||
std::unordered_set<std::string_view> available_output_columns_names;
|
||||
|
||||
for (const auto & node : actions->getNodes())
|
||||
for (const auto & node : actions->dag.getNodes())
|
||||
{
|
||||
if (available_output_columns_names.contains(node.result_name))
|
||||
continue;
|
||||
|
||||
@ -48,18 +48,18 @@ public:
|
||||
* If use_actions_nodes_as_output_columns = true output columns are initialized using actions dag nodes.
|
||||
* If additional output columns are specified they are added to output columns.
|
||||
*/
|
||||
explicit ActionsChainStep(ActionsDAGPtr actions_,
|
||||
explicit ActionsChainStep(ActionsAndProjectInputsFlagPtr actions_,
|
||||
bool use_actions_nodes_as_output_columns = true,
|
||||
ColumnsWithTypeAndName additional_output_columns_ = {});
|
||||
|
||||
/// Get actions
|
||||
ActionsDAGPtr & getActions()
|
||||
ActionsAndProjectInputsFlagPtr & getActions()
|
||||
{
|
||||
return actions;
|
||||
}
|
||||
|
||||
/// Get actions
|
||||
const ActionsDAGPtr & getActions() const
|
||||
const ActionsAndProjectInputsFlagPtr & getActions() const
|
||||
{
|
||||
return actions;
|
||||
}
|
||||
@ -98,7 +98,7 @@ public:
|
||||
private:
|
||||
void initialize();
|
||||
|
||||
ActionsDAGPtr actions;
|
||||
ActionsAndProjectInputsFlagPtr actions;
|
||||
|
||||
bool use_actions_nodes_as_output_columns = true;
|
||||
|
||||
|
||||
@ -90,7 +90,7 @@ public:
|
||||
|
||||
ActionsDAGPtr alias_column_actions_dag = std::make_shared<ActionsDAG>();
|
||||
PlannerActionsVisitor actions_visitor(planner_context, false);
|
||||
auto outputs = actions_visitor.visit(alias_column_actions_dag, column_node->getExpression());
|
||||
auto outputs = actions_visitor.visit(*alias_column_actions_dag, column_node->getExpression());
|
||||
if (outputs.size() != 1)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Expected single output in actions dag for alias column {}. Actual {}", column_node->dumpTree(), outputs.size());
|
||||
@ -340,7 +340,7 @@ void collectTableExpressionData(QueryTreeNodePtr & query_node, PlannerContextPtr
|
||||
QueryTreeNodePtr query_tree_node = query_node_typed.getPrewhere();
|
||||
|
||||
PlannerActionsVisitor visitor(planner_context, false /*use_column_identifier_as_action_node_name*/);
|
||||
auto expression_nodes = visitor.visit(prewhere_actions_dag, query_tree_node);
|
||||
auto expression_nodes = visitor.visit(*prewhere_actions_dag, query_tree_node);
|
||||
if (expression_nodes.size() != 1)
|
||||
throw Exception(ErrorCodes::ILLEGAL_PREWHERE,
|
||||
"Invalid PREWHERE. Expected single boolean expression. In query {}",
|
||||
|
||||
@ -329,12 +329,16 @@ public:
|
||||
};
|
||||
|
||||
void addExpressionStep(QueryPlan & query_plan,
|
||||
const ActionsDAGPtr & expression_actions,
|
||||
const ActionsAndProjectInputsFlagPtr & expression_actions,
|
||||
const std::string & step_description,
|
||||
std::vector<ActionsDAGPtr> & result_actions_to_execute)
|
||||
{
|
||||
result_actions_to_execute.push_back(expression_actions);
|
||||
auto expression_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), expression_actions);
|
||||
auto actions = expression_actions->dag.clone();
|
||||
if (expression_actions->project_input)
|
||||
actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
|
||||
|
||||
result_actions_to_execute.push_back(actions);
|
||||
auto expression_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), actions);
|
||||
expression_step->setStepDescription(step_description);
|
||||
query_plan.addStep(std::move(expression_step));
|
||||
}
|
||||
@ -344,9 +348,13 @@ void addFilterStep(QueryPlan & query_plan,
|
||||
const std::string & step_description,
|
||||
std::vector<ActionsDAGPtr> & result_actions_to_execute)
|
||||
{
|
||||
result_actions_to_execute.push_back(filter_analysis_result.filter_actions);
|
||||
auto actions = filter_analysis_result.filter_actions->dag.clone();
|
||||
if (filter_analysis_result.filter_actions->project_input)
|
||||
actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
|
||||
|
||||
result_actions_to_execute.push_back(actions);
|
||||
auto where_step = std::make_unique<FilterStep>(query_plan.getCurrentDataStream(),
|
||||
filter_analysis_result.filter_actions,
|
||||
actions,
|
||||
filter_analysis_result.filter_column_name,
|
||||
filter_analysis_result.remove_filter_column);
|
||||
where_step->setStepDescription(step_description);
|
||||
@ -545,14 +553,21 @@ void addTotalsHavingStep(QueryPlan & query_plan,
|
||||
const auto & having_analysis_result = expression_analysis_result.getHaving();
|
||||
bool need_finalize = !query_node.isGroupByWithRollup() && !query_node.isGroupByWithCube();
|
||||
|
||||
ActionsDAGPtr actions;
|
||||
if (having_analysis_result.filter_actions)
|
||||
result_actions_to_execute.push_back(having_analysis_result.filter_actions);
|
||||
{
|
||||
actions = having_analysis_result.filter_actions->dag.clone();
|
||||
if (having_analysis_result.filter_actions->project_input)
|
||||
actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
|
||||
|
||||
result_actions_to_execute.push_back(actions);
|
||||
}
|
||||
|
||||
auto totals_having_step = std::make_unique<TotalsHavingStep>(
|
||||
query_plan.getCurrentDataStream(),
|
||||
aggregation_analysis_result.aggregate_descriptions,
|
||||
query_analysis_result.aggregate_overflow_row,
|
||||
having_analysis_result.filter_actions,
|
||||
actions,
|
||||
having_analysis_result.filter_column_name,
|
||||
having_analysis_result.remove_filter_column,
|
||||
settings.totals_mode,
|
||||
@ -728,12 +743,12 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan,
|
||||
auto & interpolate_node_typed = interpolate_node->as<InterpolateNode &>();
|
||||
|
||||
PlannerActionsVisitor planner_actions_visitor(planner_context);
|
||||
auto expression_to_interpolate_expression_nodes = planner_actions_visitor.visit(interpolate_actions_dag,
|
||||
auto expression_to_interpolate_expression_nodes = planner_actions_visitor.visit(*interpolate_actions_dag,
|
||||
interpolate_node_typed.getExpression());
|
||||
if (expression_to_interpolate_expression_nodes.size() != 1)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expression to interpolate expected to have single action node");
|
||||
|
||||
auto interpolate_expression_nodes = planner_actions_visitor.visit(interpolate_actions_dag,
|
||||
auto interpolate_expression_nodes = planner_actions_visitor.visit(*interpolate_actions_dag,
|
||||
interpolate_node_typed.getInterpolateExpression());
|
||||
if (interpolate_expression_nodes.size() != 1)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Interpolate expression expected to have single action node");
|
||||
|
||||
@ -413,11 +413,11 @@ private:
|
||||
class ActionsScopeNode
|
||||
{
|
||||
public:
|
||||
explicit ActionsScopeNode(ActionsDAGPtr actions_dag_, QueryTreeNodePtr scope_node_)
|
||||
: actions_dag(std::move(actions_dag_))
|
||||
explicit ActionsScopeNode(ActionsDAG & actions_dag_, QueryTreeNodePtr scope_node_)
|
||||
: actions_dag(actions_dag_)
|
||||
, scope_node(std::move(scope_node_))
|
||||
{
|
||||
for (const auto & node : actions_dag->getNodes())
|
||||
for (const auto & node : actions_dag.getNodes())
|
||||
node_name_to_node[node.result_name] = &node;
|
||||
}
|
||||
|
||||
@ -456,7 +456,7 @@ public:
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"No node with name {}. There are only nodes {}",
|
||||
node_name,
|
||||
actions_dag->dumpNames());
|
||||
actions_dag.dumpNames());
|
||||
|
||||
return it->second;
|
||||
}
|
||||
@ -467,7 +467,7 @@ public:
|
||||
if (it != node_name_to_node.end())
|
||||
return it->second;
|
||||
|
||||
const auto * node = &actions_dag->addInput(node_name, column_type);
|
||||
const auto * node = &actions_dag.addInput(node_name, column_type);
|
||||
node_name_to_node[node->result_name] = node;
|
||||
|
||||
return node;
|
||||
@ -479,7 +479,7 @@ public:
|
||||
if (it != node_name_to_node.end())
|
||||
return it->second;
|
||||
|
||||
const auto * node = &actions_dag->addInput(column);
|
||||
const auto * node = &actions_dag.addInput(column);
|
||||
node_name_to_node[node->result_name] = node;
|
||||
|
||||
return node;
|
||||
@ -491,7 +491,7 @@ public:
|
||||
if (it != node_name_to_node.end())
|
||||
return it->second;
|
||||
|
||||
const auto * node = &actions_dag->addColumn(column);
|
||||
const auto * node = &actions_dag.addColumn(column);
|
||||
node_name_to_node[node->result_name] = node;
|
||||
|
||||
return node;
|
||||
@ -504,7 +504,7 @@ public:
|
||||
if (it != node_name_to_node.end())
|
||||
return it->second;
|
||||
|
||||
const auto * node = &actions_dag->addFunction(function, children, node_name);
|
||||
const auto * node = &actions_dag.addFunction(function, children, node_name);
|
||||
node_name_to_node[node->result_name] = node;
|
||||
|
||||
return node;
|
||||
@ -516,7 +516,7 @@ public:
|
||||
if (it != node_name_to_node.end())
|
||||
return it->second;
|
||||
|
||||
const auto * node = &actions_dag->addArrayJoin(*child, node_name);
|
||||
const auto * node = &actions_dag.addArrayJoin(*child, node_name);
|
||||
node_name_to_node[node->result_name] = node;
|
||||
|
||||
return node;
|
||||
@ -524,14 +524,14 @@ public:
|
||||
|
||||
private:
|
||||
std::unordered_map<std::string_view, const ActionsDAG::Node *> node_name_to_node;
|
||||
ActionsDAGPtr actions_dag;
|
||||
ActionsDAG & actions_dag;
|
||||
QueryTreeNodePtr scope_node;
|
||||
};
|
||||
|
||||
class PlannerActionsVisitorImpl
|
||||
{
|
||||
public:
|
||||
PlannerActionsVisitorImpl(ActionsDAGPtr actions_dag,
|
||||
PlannerActionsVisitorImpl(ActionsDAG & actions_dag,
|
||||
const PlannerContextPtr & planner_context_,
|
||||
bool use_column_identifier_as_action_node_name_);
|
||||
|
||||
@ -595,14 +595,14 @@ private:
|
||||
bool use_column_identifier_as_action_node_name;
|
||||
};
|
||||
|
||||
PlannerActionsVisitorImpl::PlannerActionsVisitorImpl(ActionsDAGPtr actions_dag,
|
||||
PlannerActionsVisitorImpl::PlannerActionsVisitorImpl(ActionsDAG & actions_dag,
|
||||
const PlannerContextPtr & planner_context_,
|
||||
bool use_column_identifier_as_action_node_name_)
|
||||
: planner_context(planner_context_)
|
||||
, action_node_name_helper(node_to_node_name, *planner_context, use_column_identifier_as_action_node_name_)
|
||||
, use_column_identifier_as_action_node_name(use_column_identifier_as_action_node_name_)
|
||||
{
|
||||
actions_stack.emplace_back(std::move(actions_dag), nullptr);
|
||||
actions_stack.emplace_back(actions_dag, nullptr);
|
||||
}
|
||||
|
||||
ActionsDAG::NodeRawConstPtrs PlannerActionsVisitorImpl::visit(QueryTreeNodePtr expression_node)
|
||||
@ -758,7 +758,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
|
||||
}
|
||||
|
||||
auto lambda_actions_dag = std::make_shared<ActionsDAG>();
|
||||
actions_stack.emplace_back(lambda_actions_dag, node);
|
||||
actions_stack.emplace_back(*lambda_actions_dag, node);
|
||||
|
||||
auto [lambda_expression_node_name, levels] = visitImpl(lambda_node.getExpression());
|
||||
lambda_actions_dag->getOutputs().push_back(actions_stack.back().getNodeOrThrow(lambda_expression_node_name));
|
||||
@ -886,7 +886,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
|
||||
|
||||
for (const auto & argument : function_node.getArguments())
|
||||
{
|
||||
auto index_hint_argument_expression_dag_nodes = actions_visitor.visit(index_hint_actions_dag, argument);
|
||||
auto index_hint_argument_expression_dag_nodes = actions_visitor.visit(*index_hint_actions_dag, argument);
|
||||
|
||||
for (auto & expression_dag_node : index_hint_argument_expression_dag_nodes)
|
||||
{
|
||||
@ -1013,7 +1013,7 @@ PlannerActionsVisitor::PlannerActionsVisitor(const PlannerContextPtr & planner_c
|
||||
, use_column_identifier_as_action_node_name(use_column_identifier_as_action_node_name_)
|
||||
{}
|
||||
|
||||
ActionsDAG::NodeRawConstPtrs PlannerActionsVisitor::visit(ActionsDAGPtr actions_dag, QueryTreeNodePtr expression_node)
|
||||
ActionsDAG::NodeRawConstPtrs PlannerActionsVisitor::visit(ActionsDAG & actions_dag, QueryTreeNodePtr expression_node)
|
||||
{
|
||||
PlannerActionsVisitorImpl actions_visitor_impl(actions_dag, planner_context, use_column_identifier_as_action_node_name);
|
||||
return actions_visitor_impl.visit(expression_node);
|
||||
|
||||
@ -37,7 +37,7 @@ public:
|
||||
* Necessary actions are not added in actions dag output.
|
||||
* Returns query tree expression node actions dag nodes.
|
||||
*/
|
||||
ActionsDAG::NodeRawConstPtrs visit(ActionsDAGPtr actions_dag, QueryTreeNodePtr expression_node);
|
||||
ActionsDAG::NodeRawConstPtrs visit(ActionsDAG & actions_dag, QueryTreeNodePtr expression_node);
|
||||
|
||||
private:
|
||||
const PlannerContextPtr planner_context;
|
||||
|
||||
@ -45,8 +45,10 @@ std::optional<FilterAnalysisResult> analyzeFilter(const QueryTreeNodePtr & filte
|
||||
{
|
||||
FilterAnalysisResult result;
|
||||
|
||||
result.filter_actions = buildActionsDAGFromExpressionNode(filter_expression_node, input_columns, planner_context);
|
||||
const auto * output = result.filter_actions->getOutputs().at(0);
|
||||
result.filter_actions = std::make_shared<ActionsAndProjectInputsFlag>();
|
||||
result.filter_actions->dag = buildActionsDAGFromExpressionNode(filter_expression_node, input_columns, planner_context);
|
||||
|
||||
const auto * output = result.filter_actions->dag.getOutputs().at(0);
|
||||
if (output->column && ConstantFilterDescription(*output->column).always_true)
|
||||
return {};
|
||||
|
||||
@ -116,8 +118,9 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(const QueryTreeNodeP
|
||||
|
||||
Names aggregation_keys;
|
||||
|
||||
ActionsDAGPtr before_aggregation_actions = std::make_shared<ActionsDAG>(input_columns);
|
||||
before_aggregation_actions->getOutputs().clear();
|
||||
ActionsAndProjectInputsFlagPtr before_aggregation_actions = std::make_shared<ActionsAndProjectInputsFlag>();
|
||||
before_aggregation_actions->dag = ActionsDAG(input_columns);
|
||||
before_aggregation_actions->dag.getOutputs().clear();
|
||||
|
||||
std::unordered_set<std::string_view> before_aggregation_actions_output_node_names;
|
||||
|
||||
@ -152,7 +155,7 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(const QueryTreeNodeP
|
||||
if (constant_key && !aggregates_descriptions.empty() && (!check_constants_for_group_by_key || canRemoveConstantFromGroupByKey(*constant_key)))
|
||||
continue;
|
||||
|
||||
auto expression_dag_nodes = actions_visitor.visit(before_aggregation_actions, grouping_set_key_node);
|
||||
auto expression_dag_nodes = actions_visitor.visit(before_aggregation_actions->dag, grouping_set_key_node);
|
||||
aggregation_keys.reserve(expression_dag_nodes.size());
|
||||
|
||||
for (auto & expression_dag_node : expression_dag_nodes)
|
||||
@ -165,7 +168,7 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(const QueryTreeNodeP
|
||||
auto column_after_aggregation = group_by_use_nulls && expression_dag_node->column != nullptr ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column;
|
||||
available_columns_after_aggregation.emplace_back(std::move(column_after_aggregation), expression_type_after_aggregation, expression_dag_node->result_name);
|
||||
aggregation_keys.push_back(expression_dag_node->result_name);
|
||||
before_aggregation_actions->getOutputs().push_back(expression_dag_node);
|
||||
before_aggregation_actions->dag.getOutputs().push_back(expression_dag_node);
|
||||
before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name);
|
||||
}
|
||||
}
|
||||
@ -204,7 +207,7 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(const QueryTreeNodeP
|
||||
if (constant_key && !aggregates_descriptions.empty() && (!check_constants_for_group_by_key || canRemoveConstantFromGroupByKey(*constant_key)))
|
||||
continue;
|
||||
|
||||
auto expression_dag_nodes = actions_visitor.visit(before_aggregation_actions, group_by_key_node);
|
||||
auto expression_dag_nodes = actions_visitor.visit(before_aggregation_actions->dag, group_by_key_node);
|
||||
aggregation_keys.reserve(expression_dag_nodes.size());
|
||||
|
||||
for (auto & expression_dag_node : expression_dag_nodes)
|
||||
@ -216,7 +219,7 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(const QueryTreeNodeP
|
||||
auto column_after_aggregation = group_by_use_nulls && expression_dag_node->column != nullptr ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column;
|
||||
available_columns_after_aggregation.emplace_back(std::move(column_after_aggregation), expression_type_after_aggregation, expression_dag_node->result_name);
|
||||
aggregation_keys.push_back(expression_dag_node->result_name);
|
||||
before_aggregation_actions->getOutputs().push_back(expression_dag_node);
|
||||
before_aggregation_actions->dag.getOutputs().push_back(expression_dag_node);
|
||||
before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name);
|
||||
}
|
||||
}
|
||||
@ -230,13 +233,13 @@ std::optional<AggregationAnalysisResult> analyzeAggregation(const QueryTreeNodeP
|
||||
auto & aggregate_function_node_typed = aggregate_function_node->as<FunctionNode &>();
|
||||
for (const auto & aggregate_function_node_argument : aggregate_function_node_typed.getArguments().getNodes())
|
||||
{
|
||||
auto expression_dag_nodes = actions_visitor.visit(before_aggregation_actions, aggregate_function_node_argument);
|
||||
auto expression_dag_nodes = actions_visitor.visit(before_aggregation_actions->dag, aggregate_function_node_argument);
|
||||
for (auto & expression_dag_node : expression_dag_nodes)
|
||||
{
|
||||
if (before_aggregation_actions_output_node_names.contains(expression_dag_node->result_name))
|
||||
continue;
|
||||
|
||||
before_aggregation_actions->getOutputs().push_back(expression_dag_node);
|
||||
before_aggregation_actions->dag.getOutputs().push_back(expression_dag_node);
|
||||
before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name);
|
||||
}
|
||||
}
|
||||
@ -283,8 +286,9 @@ std::optional<WindowAnalysisResult> analyzeWindow(const QueryTreeNodePtr & query
|
||||
|
||||
PlannerActionsVisitor actions_visitor(planner_context);
|
||||
|
||||
ActionsDAGPtr before_window_actions = std::make_shared<ActionsDAG>(input_columns);
|
||||
before_window_actions->getOutputs().clear();
|
||||
ActionsAndProjectInputsFlagPtr before_window_actions = std::make_shared<ActionsAndProjectInputsFlag>();
|
||||
before_window_actions->dag = ActionsDAG(input_columns);
|
||||
before_window_actions->dag.getOutputs().clear();
|
||||
|
||||
std::unordered_set<std::string_view> before_window_actions_output_node_names;
|
||||
|
||||
@ -293,25 +297,25 @@ std::optional<WindowAnalysisResult> analyzeWindow(const QueryTreeNodePtr & query
|
||||
auto & window_function_node_typed = window_function_node->as<FunctionNode &>();
|
||||
auto & window_node = window_function_node_typed.getWindowNode()->as<WindowNode &>();
|
||||
|
||||
auto expression_dag_nodes = actions_visitor.visit(before_window_actions, window_function_node_typed.getArgumentsNode());
|
||||
auto expression_dag_nodes = actions_visitor.visit(before_window_actions->dag, window_function_node_typed.getArgumentsNode());
|
||||
|
||||
for (auto & expression_dag_node : expression_dag_nodes)
|
||||
{
|
||||
if (before_window_actions_output_node_names.contains(expression_dag_node->result_name))
|
||||
continue;
|
||||
|
||||
before_window_actions->getOutputs().push_back(expression_dag_node);
|
||||
before_window_actions->dag.getOutputs().push_back(expression_dag_node);
|
||||
before_window_actions_output_node_names.insert(expression_dag_node->result_name);
|
||||
}
|
||||
|
||||
expression_dag_nodes = actions_visitor.visit(before_window_actions, window_node.getPartitionByNode());
|
||||
expression_dag_nodes = actions_visitor.visit(before_window_actions->dag, window_node.getPartitionByNode());
|
||||
|
||||
for (auto & expression_dag_node : expression_dag_nodes)
|
||||
{
|
||||
if (before_window_actions_output_node_names.contains(expression_dag_node->result_name))
|
||||
continue;
|
||||
|
||||
before_window_actions->getOutputs().push_back(expression_dag_node);
|
||||
before_window_actions->dag.getOutputs().push_back(expression_dag_node);
|
||||
before_window_actions_output_node_names.insert(expression_dag_node->result_name);
|
||||
}
|
||||
|
||||
@ -322,14 +326,14 @@ std::optional<WindowAnalysisResult> analyzeWindow(const QueryTreeNodePtr & query
|
||||
for (auto & sort_node : order_by_node_list.getNodes())
|
||||
{
|
||||
auto & sort_node_typed = sort_node->as<SortNode &>();
|
||||
expression_dag_nodes = actions_visitor.visit(before_window_actions, sort_node_typed.getExpression());
|
||||
expression_dag_nodes = actions_visitor.visit(before_window_actions->dag, sort_node_typed.getExpression());
|
||||
|
||||
for (auto & expression_dag_node : expression_dag_nodes)
|
||||
{
|
||||
if (before_window_actions_output_node_names.contains(expression_dag_node->result_name))
|
||||
continue;
|
||||
|
||||
before_window_actions->getOutputs().push_back(expression_dag_node);
|
||||
before_window_actions->dag.getOutputs().push_back(expression_dag_node);
|
||||
before_window_actions_output_node_names.insert(expression_dag_node->result_name);
|
||||
}
|
||||
}
|
||||
@ -362,7 +366,8 @@ ProjectionAnalysisResult analyzeProjection(const QueryNode & query_node,
|
||||
const PlannerContextPtr & planner_context,
|
||||
ActionsChain & actions_chain)
|
||||
{
|
||||
auto projection_actions = buildActionsDAGFromExpressionNode(query_node.getProjectionNode(), input_columns, planner_context);
|
||||
auto projection_actions = std::make_shared<ActionsAndProjectInputsFlag>();
|
||||
projection_actions->dag = buildActionsDAGFromExpressionNode(query_node.getProjectionNode(), input_columns, planner_context);
|
||||
|
||||
auto projection_columns = query_node.getProjectionColumns();
|
||||
size_t projection_columns_size = projection_columns.size();
|
||||
@ -371,7 +376,7 @@ ProjectionAnalysisResult analyzeProjection(const QueryNode & query_node,
|
||||
NamesWithAliases projection_column_names_with_display_aliases;
|
||||
projection_column_names_with_display_aliases.reserve(projection_columns_size);
|
||||
|
||||
auto & projection_actions_outputs = projection_actions->getOutputs();
|
||||
auto & projection_actions_outputs = projection_actions->dag.getOutputs();
|
||||
size_t projection_outputs_size = projection_actions_outputs.size();
|
||||
|
||||
if (projection_columns_size != projection_outputs_size)
|
||||
@ -409,8 +414,9 @@ SortAnalysisResult analyzeSort(const QueryNode & query_node,
|
||||
const PlannerContextPtr & planner_context,
|
||||
ActionsChain & actions_chain)
|
||||
{
|
||||
ActionsDAGPtr before_sort_actions = std::make_shared<ActionsDAG>(input_columns);
|
||||
auto & before_sort_actions_outputs = before_sort_actions->getOutputs();
|
||||
auto before_sort_actions = std::make_shared<ActionsAndProjectInputsFlag>();
|
||||
before_sort_actions->dag = ActionsDAG(input_columns);
|
||||
auto & before_sort_actions_outputs = before_sort_actions->dag.getOutputs();
|
||||
before_sort_actions_outputs.clear();
|
||||
|
||||
PlannerActionsVisitor actions_visitor(planner_context);
|
||||
@ -424,7 +430,7 @@ SortAnalysisResult analyzeSort(const QueryNode & query_node,
|
||||
for (const auto & sort_node : order_by_node_list.getNodes())
|
||||
{
|
||||
auto & sort_node_typed = sort_node->as<SortNode &>();
|
||||
auto expression_dag_nodes = actions_visitor.visit(before_sort_actions, sort_node_typed.getExpression());
|
||||
auto expression_dag_nodes = actions_visitor.visit(before_sort_actions->dag, sort_node_typed.getExpression());
|
||||
has_with_fill |= sort_node_typed.withFill();
|
||||
|
||||
for (auto & action_dag_node : expression_dag_nodes)
|
||||
@ -440,7 +446,7 @@ SortAnalysisResult analyzeSort(const QueryNode & query_node,
|
||||
if (has_with_fill)
|
||||
{
|
||||
for (auto & output_node : before_sort_actions_outputs)
|
||||
output_node = &before_sort_actions->materializeNode(*output_node);
|
||||
output_node = &before_sort_actions->dag.materializeNode(*output_node);
|
||||
}
|
||||
|
||||
/// We add only INPUT columns necessary for INTERPOLATE expression in before ORDER BY actions DAG
|
||||
@ -449,7 +455,7 @@ SortAnalysisResult analyzeSort(const QueryNode & query_node,
|
||||
auto & interpolate_list_node = query_node.getInterpolate()->as<ListNode &>();
|
||||
|
||||
PlannerActionsVisitor interpolate_actions_visitor(planner_context);
|
||||
auto interpolate_actions_dag = std::make_shared<ActionsDAG>();
|
||||
ActionsDAG interpolate_actions_dag;
|
||||
|
||||
for (auto & interpolate_node : interpolate_list_node.getNodes())
|
||||
{
|
||||
@ -458,10 +464,10 @@ SortAnalysisResult analyzeSort(const QueryNode & query_node,
|
||||
}
|
||||
|
||||
std::unordered_map<std::string_view, const ActionsDAG::Node *> before_sort_actions_inputs_name_to_node;
|
||||
for (const auto & node : before_sort_actions->getInputs())
|
||||
for (const auto & node : before_sort_actions->dag.getInputs())
|
||||
before_sort_actions_inputs_name_to_node.emplace(node->result_name, node);
|
||||
|
||||
for (const auto & node : interpolate_actions_dag->getNodes())
|
||||
for (const auto & node : interpolate_actions_dag.getNodes())
|
||||
{
|
||||
if (before_sort_actions_dag_output_node_names.contains(node.result_name) ||
|
||||
node.type != ActionsDAG::ActionType::INPUT)
|
||||
@ -471,7 +477,7 @@ SortAnalysisResult analyzeSort(const QueryNode & query_node,
|
||||
if (input_node_it == before_sort_actions_inputs_name_to_node.end())
|
||||
{
|
||||
auto input_column = ColumnWithTypeAndName{node.column, node.result_type, node.result_name};
|
||||
const auto * input_node = &before_sort_actions->addInput(std::move(input_column));
|
||||
const auto * input_node = &before_sort_actions->dag.addInput(std::move(input_column));
|
||||
auto [it, _] = before_sort_actions_inputs_name_to_node.emplace(node.result_name, input_node);
|
||||
input_node_it = it;
|
||||
}
|
||||
@ -496,22 +502,23 @@ LimitByAnalysisResult analyzeLimitBy(const QueryNode & query_node,
|
||||
const NameSet & required_output_nodes_names,
|
||||
ActionsChain & actions_chain)
|
||||
{
|
||||
auto before_limit_by_actions = buildActionsDAGFromExpressionNode(query_node.getLimitByNode(), input_columns, planner_context);
|
||||
auto before_limit_by_actions = std::make_shared<ActionsAndProjectInputsFlag>();
|
||||
before_limit_by_actions->dag = buildActionsDAGFromExpressionNode(query_node.getLimitByNode(), input_columns, planner_context);
|
||||
|
||||
NameSet limit_by_column_names_set;
|
||||
Names limit_by_column_names;
|
||||
limit_by_column_names.reserve(before_limit_by_actions->getOutputs().size());
|
||||
for (auto & output_node : before_limit_by_actions->getOutputs())
|
||||
limit_by_column_names.reserve(before_limit_by_actions->dag.getOutputs().size());
|
||||
for (auto & output_node : before_limit_by_actions->dag.getOutputs())
|
||||
{
|
||||
limit_by_column_names_set.insert(output_node->result_name);
|
||||
limit_by_column_names.push_back(output_node->result_name);
|
||||
}
|
||||
|
||||
for (const auto & node : before_limit_by_actions->getNodes())
|
||||
for (const auto & node : before_limit_by_actions->dag.getNodes())
|
||||
{
|
||||
if (required_output_nodes_names.contains(node.result_name) &&
|
||||
!limit_by_column_names_set.contains(node.result_name))
|
||||
before_limit_by_actions->getOutputs().push_back(&node);
|
||||
before_limit_by_actions->dag.getOutputs().push_back(&node);
|
||||
}
|
||||
|
||||
auto actions_step_before_limit_by = std::make_unique<ActionsChainStep>(before_limit_by_actions);
|
||||
@ -605,7 +612,7 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo
|
||||
if (sort_analysis_result_optional.has_value() && planner_query_processing_info.isFirstStage() && planner_query_processing_info.getToStage() != QueryProcessingStage::Complete)
|
||||
{
|
||||
const auto & before_order_by_actions = sort_analysis_result_optional->before_order_by_actions;
|
||||
for (const auto & output_node : before_order_by_actions->getOutputs())
|
||||
for (const auto & output_node : before_order_by_actions->dag.getOutputs())
|
||||
required_output_nodes_names.insert(output_node->result_name);
|
||||
}
|
||||
|
||||
@ -661,8 +668,10 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo
|
||||
}
|
||||
}
|
||||
|
||||
auto project_names_actions = std::make_shared<ActionsDAG>(project_names_input);
|
||||
project_names_actions->project(projection_analysis_result.projection_column_names_with_display_aliases);
|
||||
auto project_names_actions = std::make_shared<ActionsAndProjectInputsFlag>();
|
||||
project_names_actions->dag = ActionsDAG(project_names_input);
|
||||
project_names_actions->dag.project(projection_analysis_result.projection_column_names_with_display_aliases);
|
||||
project_names_actions->project_input = true;
|
||||
actions_chain.addStep(std::make_unique<ActionsChainStep>(project_names_actions));
|
||||
|
||||
actions_chain.finalize();
|
||||
|
||||
@ -17,22 +17,22 @@ namespace DB
|
||||
|
||||
struct ProjectionAnalysisResult
|
||||
{
|
||||
ActionsDAGPtr projection_actions;
|
||||
ActionsAndProjectInputsFlagPtr projection_actions;
|
||||
Names projection_column_names;
|
||||
NamesWithAliases projection_column_names_with_display_aliases;
|
||||
ActionsDAGPtr project_names_actions;
|
||||
ActionsAndProjectInputsFlagPtr project_names_actions;
|
||||
};
|
||||
|
||||
struct FilterAnalysisResult
|
||||
{
|
||||
ActionsDAGPtr filter_actions;
|
||||
ActionsAndProjectInputsFlagPtr filter_actions;
|
||||
std::string filter_column_name;
|
||||
bool remove_filter_column = false;
|
||||
};
|
||||
|
||||
struct AggregationAnalysisResult
|
||||
{
|
||||
ActionsDAGPtr before_aggregation_actions;
|
||||
ActionsAndProjectInputsFlagPtr before_aggregation_actions;
|
||||
Names aggregation_keys;
|
||||
AggregateDescriptions aggregate_descriptions;
|
||||
GroupingSetsParamsList grouping_sets_parameters_list;
|
||||
@ -41,19 +41,19 @@ struct AggregationAnalysisResult
|
||||
|
||||
struct WindowAnalysisResult
|
||||
{
|
||||
ActionsDAGPtr before_window_actions;
|
||||
ActionsAndProjectInputsFlagPtr before_window_actions;
|
||||
std::vector<WindowDescription> window_descriptions;
|
||||
};
|
||||
|
||||
struct SortAnalysisResult
|
||||
{
|
||||
ActionsDAGPtr before_order_by_actions;
|
||||
ActionsAndProjectInputsFlagPtr before_order_by_actions;
|
||||
bool has_with_fill = false;
|
||||
};
|
||||
|
||||
struct LimitByAnalysisResult
|
||||
{
|
||||
ActionsDAGPtr before_limit_by_actions;
|
||||
ActionsAndProjectInputsFlagPtr before_limit_by_actions;
|
||||
Names limit_by_column_names;
|
||||
};
|
||||
|
||||
|
||||
@ -1132,7 +1132,7 @@ void joinCastPlanColumnsToNullable(QueryPlan & plan_to_add_cast, PlannerContextP
|
||||
}
|
||||
}
|
||||
|
||||
cast_actions_dag->projectInput();
|
||||
cast_actions_dag->appendInputsForUnusedColumns(plan_to_add_cast.getCurrentDataStream().header);
|
||||
auto cast_join_columns_step = std::make_unique<ExpressionStep>(plan_to_add_cast.getCurrentDataStream(), std::move(cast_actions_dag));
|
||||
cast_join_columns_step->setStepDescription("Cast JOIN columns to Nullable");
|
||||
plan_to_add_cast.addStep(std::move(cast_join_columns_step));
|
||||
@ -1178,12 +1178,12 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
|
||||
join_table_expression,
|
||||
planner_context);
|
||||
|
||||
join_clauses_and_actions.left_join_expressions_actions->projectInput();
|
||||
join_clauses_and_actions.left_join_expressions_actions->appendInputsForUnusedColumns(left_plan.getCurrentDataStream().header);
|
||||
auto left_join_expressions_actions_step = std::make_unique<ExpressionStep>(left_plan.getCurrentDataStream(), join_clauses_and_actions.left_join_expressions_actions);
|
||||
left_join_expressions_actions_step->setStepDescription("JOIN actions");
|
||||
left_plan.addStep(std::move(left_join_expressions_actions_step));
|
||||
|
||||
join_clauses_and_actions.right_join_expressions_actions->projectInput();
|
||||
join_clauses_and_actions.right_join_expressions_actions->appendInputsForUnusedColumns(right_plan.getCurrentDataStream().header);
|
||||
auto right_join_expressions_actions_step = std::make_unique<ExpressionStep>(right_plan.getCurrentDataStream(), join_clauses_and_actions.right_join_expressions_actions);
|
||||
right_join_expressions_actions_step->setStepDescription("JOIN actions");
|
||||
right_plan.addStep(std::move(right_join_expressions_actions_step));
|
||||
@ -1235,7 +1235,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
|
||||
output_node = &cast_actions_dag->addCast(*output_node, cast_type, output_node->result_name);
|
||||
}
|
||||
|
||||
cast_actions_dag->projectInput();
|
||||
cast_actions_dag->appendInputsForUnusedColumns(plan_to_add_cast.getCurrentDataStream().header);
|
||||
auto cast_join_columns_step
|
||||
= std::make_unique<ExpressionStep>(plan_to_add_cast.getCurrentDataStream(), std::move(cast_actions_dag));
|
||||
cast_join_columns_step->setStepDescription("Cast JOIN USING columns");
|
||||
@ -1630,7 +1630,7 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_
|
||||
array_join_column_names.insert(array_join_column_identifier);
|
||||
|
||||
auto & array_join_expression_column = array_join_expression->as<ColumnNode &>();
|
||||
auto expression_dag_index_nodes = actions_visitor.visit(array_join_action_dag, array_join_expression_column.getExpressionOrThrow());
|
||||
auto expression_dag_index_nodes = actions_visitor.visit(*array_join_action_dag, array_join_expression_column.getExpressionOrThrow());
|
||||
|
||||
for (auto & expression_dag_index_node : expression_dag_index_nodes)
|
||||
{
|
||||
@ -1640,7 +1640,7 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_
|
||||
}
|
||||
}
|
||||
|
||||
array_join_action_dag->projectInput();
|
||||
array_join_action_dag->appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
|
||||
|
||||
join_tree_query_plan.actions_dags.push_back(array_join_action_dag);
|
||||
|
||||
|
||||
@ -183,7 +183,7 @@ const ActionsDAG::Node * appendExpression(
|
||||
const JoinNode & join_node)
|
||||
{
|
||||
PlannerActionsVisitor join_expression_visitor(planner_context);
|
||||
auto join_expression_dag_node_raw_pointers = join_expression_visitor.visit(dag, expression);
|
||||
auto join_expression_dag_node_raw_pointers = join_expression_visitor.visit(*dag, expression);
|
||||
if (join_expression_dag_node_raw_pointers.size() != 1)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"JOIN {} ON clause contains multiple expressions",
|
||||
@ -603,7 +603,7 @@ JoinClausesAndActions buildJoinClausesAndActions(
|
||||
{
|
||||
auto mixed_join_expressions_actions = std::make_shared<ActionsDAG>(mixed_table_expression_columns);
|
||||
PlannerActionsVisitor join_expression_visitor(planner_context);
|
||||
auto join_expression_dag_node_raw_pointers = join_expression_visitor.visit(mixed_join_expressions_actions, join_expression);
|
||||
auto join_expression_dag_node_raw_pointers = join_expression_visitor.visit(*mixed_join_expressions_actions, join_expression);
|
||||
if (join_expression_dag_node_raw_pointers.size() != 1)
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR, "JOIN {} ON clause contains multiple expressions", join_node.formatASTForErrorMessage());
|
||||
|
||||
@ -213,14 +213,14 @@ StorageLimits buildStorageLimits(const Context & context, const SelectQueryOptio
|
||||
return {limits, leaf_limits};
|
||||
}
|
||||
|
||||
ActionsDAGPtr buildActionsDAGFromExpressionNode(const QueryTreeNodePtr & expression_node,
|
||||
ActionsDAG buildActionsDAGFromExpressionNode(const QueryTreeNodePtr & expression_node,
|
||||
const ColumnsWithTypeAndName & input_columns,
|
||||
const PlannerContextPtr & planner_context)
|
||||
{
|
||||
ActionsDAGPtr action_dag = std::make_shared<ActionsDAG>(input_columns);
|
||||
ActionsDAG action_dag(input_columns);
|
||||
PlannerActionsVisitor actions_visitor(planner_context);
|
||||
auto expression_dag_index_nodes = actions_visitor.visit(action_dag, expression_node);
|
||||
action_dag->getOutputs() = std::move(expression_dag_index_nodes);
|
||||
action_dag.getOutputs() = std::move(expression_dag_index_nodes);
|
||||
|
||||
return action_dag;
|
||||
}
|
||||
@ -443,7 +443,7 @@ FilterDAGInfo buildFilterInfo(QueryTreeNodePtr filter_query_tree,
|
||||
auto filter_actions_dag = std::make_shared<ActionsDAG>();
|
||||
|
||||
PlannerActionsVisitor actions_visitor(planner_context, false /*use_column_identifier_as_action_node_name*/);
|
||||
auto expression_nodes = actions_visitor.visit(filter_actions_dag, filter_query_tree);
|
||||
auto expression_nodes = actions_visitor.visit(*filter_actions_dag, filter_query_tree);
|
||||
if (expression_nodes.size() != 1)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Filter actions must return single output node. Actual {}",
|
||||
|
||||
@ -47,7 +47,7 @@ StorageLimits buildStorageLimits(const Context & context, const SelectQueryOptio
|
||||
* Inputs are not used for actions dag outputs.
|
||||
* Only root query tree expression node is used as actions dag output.
|
||||
*/
|
||||
ActionsDAGPtr buildActionsDAGFromExpressionNode(const QueryTreeNodePtr & expression_node,
|
||||
ActionsDAG buildActionsDAGFromExpressionNode(const QueryTreeNodePtr & expression_node,
|
||||
const ColumnsWithTypeAndName & input_columns,
|
||||
const PlannerContextPtr & planner_context);
|
||||
|
||||
|
||||
@ -2,10 +2,25 @@
|
||||
#include <Processors/QueryPlan/FilterStep.h>
|
||||
#include <Processors/QueryPlan/ExpressionStep.h>
|
||||
#include <Interpreters/ActionsDAG.h>
|
||||
#include <Functions/FunctionsLogical.h>
|
||||
#include <Functions/IFunctionAdaptors.h>
|
||||
|
||||
namespace DB::QueryPlanOptimizations
|
||||
{
|
||||
|
||||
static void removeFromOutputs(ActionsDAG & dag, const ActionsDAG::Node & node)
|
||||
{
|
||||
auto & outputs = dag.getOutputs();
|
||||
for (size_t i = 0; i < outputs.size(); ++i)
|
||||
{
|
||||
if (&node == outputs[i])
|
||||
{
|
||||
outputs.erase(outputs.begin() + i);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size_t tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &)
|
||||
{
|
||||
if (parent_node->children.size() != 1)
|
||||
@ -19,6 +34,7 @@ size_t tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &)
|
||||
auto * parent_expr = typeid_cast<ExpressionStep *>(parent.get());
|
||||
auto * parent_filter = typeid_cast<FilterStep *>(parent.get());
|
||||
auto * child_expr = typeid_cast<ExpressionStep *>(child.get());
|
||||
auto * child_filter = typeid_cast<FilterStep *>(child.get());
|
||||
|
||||
if (parent_expr && child_expr)
|
||||
{
|
||||
@ -60,6 +76,42 @@ size_t tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &)
|
||||
parent_node->children.swap(child_node->children);
|
||||
return 1;
|
||||
}
|
||||
else if (parent_filter && child_filter)
|
||||
{
|
||||
const auto & child_actions = child_filter->getExpression();
|
||||
const auto & parent_actions = parent_filter->getExpression();
|
||||
|
||||
if (child_actions->hasArrayJoin())
|
||||
return 0;
|
||||
|
||||
auto actions = child_actions->clone();
|
||||
const auto & child_filter_node = actions->findInOutputs(child_filter->getFilterColumnName());
|
||||
if (child_filter->removesFilterColumn())
|
||||
removeFromOutputs(*actions, child_filter_node);
|
||||
|
||||
actions->mergeInplace(std::move(*parent_actions->clone()));
|
||||
|
||||
const auto & parent_filter_node = actions->findInOutputs(parent_filter->getFilterColumnName());
|
||||
if (parent_filter->removesFilterColumn())
|
||||
removeFromOutputs(*actions, parent_filter_node);
|
||||
|
||||
FunctionOverloadResolverPtr func_builder_and = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionAnd>());
|
||||
const auto & condition = actions->addFunction(func_builder_and, {&child_filter_node, &parent_filter_node}, {});
|
||||
auto & outputs = actions->getOutputs();
|
||||
outputs.insert(outputs.begin(), &condition);
|
||||
|
||||
actions->removeUnusedActions(false);
|
||||
|
||||
auto filter = std::make_unique<FilterStep>(child_filter->getInputStreams().front(),
|
||||
actions,
|
||||
condition.result_name,
|
||||
true);
|
||||
filter->setStepDescription("(" + parent_filter->getStepDescription() + " + " + child_filter->getStepDescription() + ")");
|
||||
|
||||
parent_node->step = std::move(filter);
|
||||
parent_node->children.swap(child_node->children);
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -176,8 +176,6 @@ static void appendExpression(ActionsDAGPtr & dag, const ActionsDAGPtr & expressi
|
||||
dag->mergeInplace(std::move(*expression->clone()));
|
||||
else
|
||||
dag = expression->clone();
|
||||
|
||||
dag->projectInput(false);
|
||||
}
|
||||
|
||||
/// This function builds a common DAG which is a merge of DAGs from Filter and Expression steps chain.
|
||||
|
||||
@ -77,7 +77,7 @@ static AggregateProjectionInfo getAggregatingProjectionInfo(
|
||||
|
||||
AggregateProjectionInfo info;
|
||||
info.context = interpreter.getContext();
|
||||
info.before_aggregation = analysis_result.before_aggregation;
|
||||
info.before_aggregation = analysis_result.before_aggregation->dag.clone();
|
||||
info.keys = query_analyzer->aggregationKeys().getNames();
|
||||
info.aggregates = query_analyzer->aggregates();
|
||||
|
||||
|
||||
@ -849,10 +849,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_
|
||||
|
||||
static ActionsDAGPtr createProjection(const Block & header)
|
||||
{
|
||||
auto projection = std::make_shared<ActionsDAG>(header.getNamesAndTypesList());
|
||||
projection->removeUnusedActions(header.getNames());
|
||||
projection->projectInput();
|
||||
return projection;
|
||||
return std::make_shared<ActionsDAG>(header.getNamesAndTypesList());
|
||||
}
|
||||
|
||||
Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder(
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
#include <Processors/Sinks/SinkToStorage.h>
|
||||
#include <Processors/Transforms/ExpressionTransform.h>
|
||||
#include <Processors/Transforms/MaterializingTransform.h>
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
#include <Processors/Executors/PullingAsyncPipelineExecutor.h>
|
||||
|
||||
#include <QueryPipeline/QueryPipelineBuilder.h>
|
||||
|
||||
@ -178,7 +178,7 @@ void AddingDefaultsTransform::transform(Chunk & chunk)
|
||||
auto dag = evaluateMissingDefaults(evaluate_block, header.getNamesAndTypesList(), columns, context, false);
|
||||
if (dag)
|
||||
{
|
||||
auto actions = std::make_shared<ExpressionActions>(std::move(dag), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes));
|
||||
auto actions = std::make_shared<ExpressionActions>(std::move(dag), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes), true);
|
||||
actions->execute(evaluate_block);
|
||||
}
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/formatReadable.h>
|
||||
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
|
||||
|
||||
namespace ProfileEvents
|
||||
@ -783,7 +783,7 @@ void AggregatingTransform::initGenerate()
|
||||
{
|
||||
/// Just a reasonable constant, matches default value for the setting `preferred_block_size_bytes`
|
||||
static constexpr size_t oneMB = 1024 * 1024;
|
||||
return std::make_shared<SimpleSquashingChunksTransform>(header, params->params.max_block_size, oneMB);
|
||||
return std::make_shared<SimpleSquashingTransform>(header, params->params.max_block_size, oneMB);
|
||||
});
|
||||
}
|
||||
/// AggregatingTransform::expandPipeline expects single output port.
|
||||
|
||||
63
src/Processors/Transforms/ApplySquashingTransform.h
Normal file
63
src/Processors/Transforms/ApplySquashingTransform.h
Normal file
@ -0,0 +1,63 @@
|
||||
#pragma once
|
||||
#include <Interpreters/Squashing.h>
|
||||
#include <Processors/ISimpleTransform.h>
|
||||
#include <Processors/Sinks/SinkToStorage.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class ApplySquashingTransform : public ExceptionKeepingTransform
|
||||
{
|
||||
public:
|
||||
explicit ApplySquashingTransform(const Block & header, const size_t min_block_size_rows, const size_t min_block_size_bytes)
|
||||
: ExceptionKeepingTransform(header, header, false)
|
||||
, squashing(header, min_block_size_rows, min_block_size_bytes)
|
||||
{
|
||||
}
|
||||
|
||||
String getName() const override { return "ApplySquashingTransform"; }
|
||||
|
||||
void work() override
|
||||
{
|
||||
if (stage == Stage::Exception)
|
||||
{
|
||||
data.chunk.clear();
|
||||
ready_input = false;
|
||||
return;
|
||||
}
|
||||
|
||||
ExceptionKeepingTransform::work();
|
||||
if (finish_chunk)
|
||||
{
|
||||
data.chunk = std::move(finish_chunk);
|
||||
ready_output = true;
|
||||
}
|
||||
}
|
||||
|
||||
protected:
|
||||
void onConsume(Chunk chunk) override
|
||||
{
|
||||
if (auto res_chunk = DB::Squashing::squash(std::move(chunk)))
|
||||
cur_chunk.setColumns(res_chunk.getColumns(), res_chunk.getNumRows());
|
||||
}
|
||||
|
||||
GenerateResult onGenerate() override
|
||||
{
|
||||
GenerateResult res;
|
||||
res.chunk = std::move(cur_chunk);
|
||||
res.is_done = true;
|
||||
return res;
|
||||
}
|
||||
void onFinish() override
|
||||
{
|
||||
auto chunk = DB::Squashing::squash({});
|
||||
finish_chunk.setColumns(chunk.getColumns(), chunk.getNumRows());
|
||||
}
|
||||
|
||||
private:
|
||||
Squashing squashing;
|
||||
Chunk cur_chunk;
|
||||
Chunk finish_chunk;
|
||||
};
|
||||
|
||||
}
|
||||
145
src/Processors/Transforms/PlanSquashingTransform.cpp
Normal file
145
src/Processors/Transforms/PlanSquashingTransform.cpp
Normal file
@ -0,0 +1,145 @@
|
||||
#include <Processors/Transforms/PlanSquashingTransform.h>
|
||||
#include <Processors/IProcessor.h>
|
||||
#include <Common/Exception.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
PlanSquashingTransform::PlanSquashingTransform(const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes, size_t num_ports)
|
||||
: IProcessor(InputPorts(num_ports, header), OutputPorts(num_ports, header)), squashing(header, min_block_size_rows, min_block_size_bytes)
|
||||
{
|
||||
}
|
||||
|
||||
IProcessor::Status PlanSquashingTransform::prepare()
|
||||
{
|
||||
Status status = Status::Ready;
|
||||
|
||||
while (planning_status != PlanningStatus::FINISH)
|
||||
{
|
||||
switch (planning_status)
|
||||
{
|
||||
case INIT:
|
||||
init();
|
||||
break;
|
||||
case READ_IF_CAN:
|
||||
return prepareConsume();
|
||||
case PUSH:
|
||||
return sendOrFlush();
|
||||
case FLUSH:
|
||||
return sendOrFlush();
|
||||
case FINISH:
|
||||
break; /// never reached
|
||||
}
|
||||
}
|
||||
if (status == Status::Ready)
|
||||
status = finish();
|
||||
else
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "There should be a Ready status to finish the PlanSquashing");
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
void PlanSquashingTransform::work()
|
||||
{
|
||||
prepare();
|
||||
}
|
||||
|
||||
void PlanSquashingTransform::init()
|
||||
{
|
||||
for (auto input: inputs)
|
||||
if (!input.isFinished())
|
||||
input.setNeeded();
|
||||
|
||||
planning_status = PlanningStatus::READ_IF_CAN;
|
||||
}
|
||||
|
||||
IProcessor::Status PlanSquashingTransform::prepareConsume()
|
||||
{
|
||||
bool all_finished = true;
|
||||
for (auto & input : inputs)
|
||||
{
|
||||
if (!input.isFinished())
|
||||
{
|
||||
all_finished = false;
|
||||
input.setNeeded();
|
||||
}
|
||||
else
|
||||
continue;
|
||||
|
||||
if (input.hasData())
|
||||
{
|
||||
chunk = input.pull();
|
||||
chunk = transform(std::move(chunk));
|
||||
|
||||
if (chunk.hasChunkInfo())
|
||||
{
|
||||
planning_status = PlanningStatus::PUSH;
|
||||
return Status::Ready;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (all_finished) /// If all inputs are closed, we check if we have data in balancing
|
||||
{
|
||||
if (squashing.isDataLeft()) /// If we have data in balancing, we process this data
|
||||
{
|
||||
planning_status = PlanningStatus::FLUSH;
|
||||
chunk = flushChunk();
|
||||
return Status::Ready;
|
||||
}
|
||||
planning_status = PlanningStatus::FINISH;
|
||||
return Status::Ready;
|
||||
}
|
||||
|
||||
return Status::NeedData;
|
||||
}
|
||||
|
||||
Chunk PlanSquashingTransform::transform(Chunk && chunk_)
|
||||
{
|
||||
return squashing.add(std::move(chunk_));
|
||||
}
|
||||
|
||||
Chunk PlanSquashingTransform::flushChunk()
|
||||
{
|
||||
return squashing.flush();
|
||||
}
|
||||
|
||||
IProcessor::Status PlanSquashingTransform::sendOrFlush()
|
||||
{
|
||||
if (!chunk)
|
||||
{
|
||||
planning_status = PlanningStatus::FINISH;
|
||||
return Status::Ready;
|
||||
}
|
||||
|
||||
for (auto &output : outputs)
|
||||
{
|
||||
if (output.canPush())
|
||||
{
|
||||
if (planning_status == PlanningStatus::PUSH)
|
||||
planning_status = PlanningStatus::READ_IF_CAN;
|
||||
else
|
||||
planning_status = PlanningStatus::FINISH;
|
||||
|
||||
output.push(std::move(chunk));
|
||||
return Status::Ready;
|
||||
}
|
||||
}
|
||||
return Status::PortFull;
|
||||
}
|
||||
|
||||
IProcessor::Status PlanSquashingTransform::finish()
|
||||
{
|
||||
for (auto & in : inputs)
|
||||
in.close();
|
||||
for (auto & output : outputs)
|
||||
output.finish();
|
||||
|
||||
return Status::Finished;
|
||||
}
|
||||
}
|
||||
47
src/Processors/Transforms/PlanSquashingTransform.h
Normal file
47
src/Processors/Transforms/PlanSquashingTransform.h
Normal file
@ -0,0 +1,47 @@
|
||||
#pragma once
|
||||
|
||||
#include <Processors/Sinks/SinkToStorage.h>
|
||||
#include <Processors/IProcessor.h>
|
||||
#include <Interpreters/Squashing.h>
|
||||
|
||||
enum PlanningStatus
|
||||
{
|
||||
INIT,
|
||||
READ_IF_CAN,
|
||||
PUSH,
|
||||
FLUSH,
|
||||
FINISH
|
||||
};
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class PlanSquashingTransform : public IProcessor
|
||||
{
|
||||
public:
|
||||
PlanSquashingTransform(
|
||||
const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes, size_t num_ports);
|
||||
|
||||
String getName() const override { return "PlanSquashingTransform"; }
|
||||
|
||||
InputPorts & getInputPorts() { return inputs; }
|
||||
OutputPorts & getOutputPorts() { return outputs; }
|
||||
|
||||
Status prepare() override;
|
||||
void work() override;
|
||||
void init();
|
||||
Status prepareConsume();
|
||||
Status sendOrFlush();
|
||||
Status waitForDataIn();
|
||||
Status finish();
|
||||
|
||||
Chunk transform(Chunk && chunk);
|
||||
Chunk flushChunk();
|
||||
|
||||
private:
|
||||
Chunk chunk;
|
||||
Squashing squashing;
|
||||
PlanningStatus planning_status = PlanningStatus::INIT;
|
||||
};
|
||||
}
|
||||
|
||||
@ -1,94 +0,0 @@
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
SquashingChunksTransform::SquashingChunksTransform(
|
||||
const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes)
|
||||
: ExceptionKeepingTransform(header, header, false)
|
||||
, squashing(min_block_size_rows, min_block_size_bytes)
|
||||
{
|
||||
}
|
||||
|
||||
void SquashingChunksTransform::onConsume(Chunk chunk)
|
||||
{
|
||||
if (auto block = squashing.add(getInputPort().getHeader().cloneWithColumns(chunk.detachColumns())))
|
||||
{
|
||||
cur_chunk.setColumns(block.getColumns(), block.rows());
|
||||
}
|
||||
}
|
||||
|
||||
SquashingChunksTransform::GenerateResult SquashingChunksTransform::onGenerate()
|
||||
{
|
||||
GenerateResult res;
|
||||
res.chunk = std::move(cur_chunk);
|
||||
res.is_done = true;
|
||||
return res;
|
||||
}
|
||||
|
||||
void SquashingChunksTransform::onFinish()
|
||||
{
|
||||
auto block = squashing.add({});
|
||||
finish_chunk.setColumns(block.getColumns(), block.rows());
|
||||
}
|
||||
|
||||
void SquashingChunksTransform::work()
|
||||
{
|
||||
if (stage == Stage::Exception)
|
||||
{
|
||||
data.chunk.clear();
|
||||
ready_input = false;
|
||||
return;
|
||||
}
|
||||
|
||||
ExceptionKeepingTransform::work();
|
||||
if (finish_chunk)
|
||||
{
|
||||
data.chunk = std::move(finish_chunk);
|
||||
ready_output = true;
|
||||
}
|
||||
}
|
||||
|
||||
SimpleSquashingChunksTransform::SimpleSquashingChunksTransform(
|
||||
const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes)
|
||||
: IInflatingTransform(header, header), squashing(min_block_size_rows, min_block_size_bytes)
|
||||
{
|
||||
}
|
||||
|
||||
void SimpleSquashingChunksTransform::consume(Chunk chunk)
|
||||
{
|
||||
Block current_block = squashing.add(getInputPort().getHeader().cloneWithColumns(chunk.detachColumns()));
|
||||
squashed_chunk.setColumns(current_block.getColumns(), current_block.rows());
|
||||
}
|
||||
|
||||
Chunk SimpleSquashingChunksTransform::generate()
|
||||
{
|
||||
if (squashed_chunk.empty())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't generate chunk in SimpleSquashingChunksTransform");
|
||||
|
||||
Chunk result_chunk;
|
||||
result_chunk.swap(squashed_chunk);
|
||||
return result_chunk;
|
||||
}
|
||||
|
||||
bool SimpleSquashingChunksTransform::canGenerate()
|
||||
{
|
||||
return !squashed_chunk.empty();
|
||||
}
|
||||
|
||||
Chunk SimpleSquashingChunksTransform::getRemaining()
|
||||
{
|
||||
Block current_block = squashing.add({});
|
||||
squashed_chunk.setColumns(current_block.getColumns(), current_block.rows());
|
||||
|
||||
Chunk result_chunk;
|
||||
result_chunk.swap(squashed_chunk);
|
||||
return result_chunk;
|
||||
}
|
||||
|
||||
}
|
||||
108
src/Processors/Transforms/SquashingTransform.cpp
Normal file
108
src/Processors/Transforms/SquashingTransform.cpp
Normal file
@ -0,0 +1,108 @@
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
#include <Interpreters/Squashing.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
SquashingTransform::SquashingTransform(
|
||||
const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes)
|
||||
: ExceptionKeepingTransform(header, header, false)
|
||||
, squashing(header, min_block_size_rows, min_block_size_bytes)
|
||||
{
|
||||
}
|
||||
|
||||
void SquashingTransform::onConsume(Chunk chunk)
|
||||
{
|
||||
Chunk planned_chunk = squashing.add(std::move(chunk));
|
||||
if (planned_chunk.hasChunkInfo())
|
||||
cur_chunk = DB::Squashing::squash(std::move(planned_chunk));
|
||||
}
|
||||
|
||||
SquashingTransform::GenerateResult SquashingTransform::onGenerate()
|
||||
{
|
||||
GenerateResult res;
|
||||
res.chunk = std::move(cur_chunk);
|
||||
res.is_done = true;
|
||||
return res;
|
||||
}
|
||||
|
||||
void SquashingTransform::onFinish()
|
||||
{
|
||||
Chunk chunk = squashing.flush();
|
||||
if (chunk.hasChunkInfo())
|
||||
chunk = DB::Squashing::squash(std::move(chunk));
|
||||
finish_chunk.setColumns(chunk.getColumns(), chunk.getNumRows());
|
||||
}
|
||||
|
||||
void SquashingTransform::work()
|
||||
{
|
||||
if (stage == Stage::Exception)
|
||||
{
|
||||
data.chunk.clear();
|
||||
ready_input = false;
|
||||
return;
|
||||
}
|
||||
|
||||
ExceptionKeepingTransform::work();
|
||||
if (finish_chunk)
|
||||
{
|
||||
data.chunk = std::move(finish_chunk);
|
||||
ready_output = true;
|
||||
}
|
||||
}
|
||||
|
||||
SimpleSquashingTransform::SimpleSquashingTransform(
|
||||
const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes)
|
||||
: ISimpleTransform(header, header, false)
|
||||
, squashing(header, min_block_size_rows, min_block_size_bytes)
|
||||
{
|
||||
}
|
||||
|
||||
void SimpleSquashingTransform::transform(Chunk & chunk)
|
||||
{
|
||||
if (!finished)
|
||||
{
|
||||
Chunk planned_chunk = squashing.add(std::move(chunk));
|
||||
if (planned_chunk.hasChunkInfo())
|
||||
chunk = DB::Squashing::squash(std::move(planned_chunk));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (chunk.hasRows())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk expected to be empty, otherwise it will be lost");
|
||||
|
||||
chunk = squashing.flush();
|
||||
if (chunk.hasChunkInfo())
|
||||
chunk = DB::Squashing::squash(std::move(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
IProcessor::Status SimpleSquashingTransform::prepare()
|
||||
{
|
||||
if (!finished && input.isFinished())
|
||||
{
|
||||
if (output.isFinished())
|
||||
return Status::Finished;
|
||||
|
||||
if (!output.canPush())
|
||||
return Status::PortFull;
|
||||
|
||||
if (has_output)
|
||||
{
|
||||
output.pushData(std::move(output_data));
|
||||
has_output = false;
|
||||
return Status::PortFull;
|
||||
}
|
||||
|
||||
finished = true;
|
||||
/// On the next call to transform() we will return all data buffered in `squashing` (if any)
|
||||
return Status::Ready;
|
||||
}
|
||||
return ISimpleTransform::prepare();
|
||||
}
|
||||
}
|
||||
@ -1,17 +1,17 @@
|
||||
#pragma once
|
||||
|
||||
#include <Interpreters/SquashingTransform.h>
|
||||
#include <Interpreters/Squashing.h>
|
||||
#include <Processors/ISimpleTransform.h>
|
||||
#include <Processors/IInflatingTransform.h>
|
||||
#include <Processors/Sinks/SinkToStorage.h>
|
||||
#include <Processors/Transforms/ApplySquashingTransform.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class SquashingChunksTransform : public ExceptionKeepingTransform
|
||||
class SquashingTransform : public ExceptionKeepingTransform
|
||||
{
|
||||
public:
|
||||
explicit SquashingChunksTransform(
|
||||
explicit SquashingTransform(
|
||||
const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes);
|
||||
|
||||
String getName() const override { return "SquashingTransform"; }
|
||||
@ -24,28 +24,27 @@ protected:
|
||||
void onFinish() override;
|
||||
|
||||
private:
|
||||
SquashingTransform squashing;
|
||||
Squashing squashing;
|
||||
Chunk cur_chunk;
|
||||
Chunk finish_chunk;
|
||||
};
|
||||
|
||||
/// Doesn't care about propagating exceptions and thus doesn't throw LOGICAL_ERROR if the following transform closes its input port.
|
||||
class SimpleSquashingChunksTransform : public IInflatingTransform
|
||||
class SimpleSquashingTransform : public ISimpleTransform
|
||||
{
|
||||
public:
|
||||
explicit SimpleSquashingChunksTransform(const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes);
|
||||
explicit SimpleSquashingTransform(const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes);
|
||||
|
||||
String getName() const override { return "SimpleSquashingTransform"; }
|
||||
|
||||
protected:
|
||||
void consume(Chunk chunk) override;
|
||||
bool canGenerate() override;
|
||||
Chunk generate() override;
|
||||
Chunk getRemaining() override;
|
||||
void transform(Chunk &) override;
|
||||
|
||||
IProcessor::Status prepare() override;
|
||||
|
||||
private:
|
||||
SquashingTransform squashing;
|
||||
Chunk squashed_chunk;
|
||||
};
|
||||
Squashing squashing;
|
||||
|
||||
bool finished = false;
|
||||
};
|
||||
}
|
||||
@ -6,7 +6,8 @@
|
||||
#include <Interpreters/InterpreterSelectQueryAnalyzer.h>
|
||||
#include <Parsers/ASTInsertQuery.h>
|
||||
#include <Processors/Transforms/CountingTransform.h>
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
#include <Processors/Transforms/PlanSquashingTransform.h>
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
#include <Processors/Transforms/ExpressionTransform.h>
|
||||
#include <Processors/Executors/PullingPipelineExecutor.h>
|
||||
#include <Storages/LiveView/StorageLiveView.h>
|
||||
@ -371,7 +372,7 @@ std::optional<Chain> generateViewChain(
|
||||
bool table_prefers_large_blocks = inner_table->prefersLargeBlocks();
|
||||
const auto & settings = insert_context->getSettingsRef();
|
||||
|
||||
out.addSource(std::make_shared<SquashingChunksTransform>(
|
||||
out.addSource(std::make_shared<SquashingTransform>(
|
||||
out.getInputHeader(),
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL));
|
||||
@ -622,7 +623,7 @@ static QueryPipeline process(Block block, ViewRuntimeData & view, const ViewsDat
|
||||
/// Squashing is needed here because the materialized view query can generate a lot of blocks
|
||||
/// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
|
||||
/// and two-level aggregation is triggered).
|
||||
pipeline.addTransform(std::make_shared<SquashingChunksTransform>(
|
||||
pipeline.addTransform(std::make_shared<SquashingTransform>(
|
||||
pipeline.getHeader(),
|
||||
context->getSettingsRef().min_insert_block_size_rows,
|
||||
context->getSettingsRef().min_insert_block_size_bytes));
|
||||
|
||||
@ -1046,12 +1046,21 @@ void HTTPHandler::formatExceptionForClient(int exception_code, HTTPServerRequest
|
||||
|
||||
/// FIXME: make sure that no one else is reading from the same stream at the moment.
|
||||
|
||||
/// If HTTP method is POST and Keep-Alive is turned on, we should read the whole request body
|
||||
/// If HTTP method is POST and Keep-Alive is turned on, we should try to read the whole request body
|
||||
/// to avoid reading part of the current request body in the next request.
|
||||
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && response.getKeepAlive()
|
||||
&& exception_code != ErrorCodes::HTTP_LENGTH_REQUIRED && !request.getStream().eof())
|
||||
&& exception_code != ErrorCodes::HTTP_LENGTH_REQUIRED)
|
||||
{
|
||||
request.getStream().ignoreAll();
|
||||
try
|
||||
{
|
||||
if (!request.getStream().eof())
|
||||
request.getStream().ignoreAll();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(log, "Cannot read remaining request body during exception handling");
|
||||
response.setKeepAlive(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (exception_code == ErrorCodes::REQUIRED_PASSWORD)
|
||||
@ -1063,7 +1072,6 @@ void HTTPHandler::formatExceptionForClient(int exception_code, HTTPServerRequest
|
||||
void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & write_event)
|
||||
{
|
||||
setThreadName("HTTPHandler");
|
||||
ThreadStatus thread_status;
|
||||
|
||||
session = std::make_unique<Session>(server.context(), ClientInfo::Interface::HTTP, request.isSecure());
|
||||
SCOPE_EXIT({ session.reset(); });
|
||||
|
||||
@ -8,7 +8,6 @@
|
||||
#include <Interpreters/InterserverIOHandler.h>
|
||||
#include <Server/HTTP/HTMLForm.h>
|
||||
#include <Server/HTTP/WriteBufferFromHTTPServerResponse.h>
|
||||
#include <Common/ThreadStatus.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/setThreadName.h>
|
||||
|
||||
@ -81,7 +80,6 @@ void InterserverIOHTTPHandler::processQuery(HTTPServerRequest & request, HTTPSer
|
||||
void InterserverIOHTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & write_event)
|
||||
{
|
||||
setThreadName("IntersrvHandler");
|
||||
ThreadStatus thread_status;
|
||||
|
||||
/// In order to work keep-alive.
|
||||
if (request.getVersion() == HTTPServerRequest::HTTP_1_1)
|
||||
|
||||
@ -309,7 +309,6 @@ Poco::Timespan KeeperTCPHandler::receiveHandshake(int32_t handshake_length, bool
|
||||
void KeeperTCPHandler::runImpl()
|
||||
{
|
||||
setThreadName("KeeperHandler");
|
||||
ThreadStatus thread_status;
|
||||
|
||||
socket().setReceiveTimeout(receive_timeout);
|
||||
socket().setSendTimeout(send_timeout);
|
||||
|
||||
@ -24,7 +24,6 @@
|
||||
#include <Common/CurrentThread.h>
|
||||
#include <Common/NetException.h>
|
||||
#include <Common/OpenSSLHelpers.h>
|
||||
#include <Common/ThreadStatus.h>
|
||||
#include <Common/config_version.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/re2.h>
|
||||
@ -199,7 +198,6 @@ MySQLHandler::~MySQLHandler() = default;
|
||||
void MySQLHandler::run()
|
||||
{
|
||||
setThreadName("MySQLHandler");
|
||||
ThreadStatus thread_status;
|
||||
|
||||
session = std::make_unique<Session>(server.context(), ClientInfo::Interface::MYSQL);
|
||||
SCOPE_EXIT({ session.reset(); });
|
||||
|
||||
@ -10,7 +10,6 @@
|
||||
#include <base/scope_guard.h>
|
||||
#include <pcg_random.hpp>
|
||||
#include <Common/CurrentThread.h>
|
||||
#include <Common/ThreadStatus.h>
|
||||
#include <Common/config_version.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <Common/setThreadName.h>
|
||||
@ -59,7 +58,6 @@ void PostgreSQLHandler::changeIO(Poco::Net::StreamSocket & socket)
|
||||
void PostgreSQLHandler::run()
|
||||
{
|
||||
setThreadName("PostgresHandler");
|
||||
ThreadStatus thread_status;
|
||||
|
||||
session = std::make_unique<Session>(server.context(), ClientInfo::Interface::POSTGRESQL);
|
||||
SCOPE_EXIT({ session.reset(); });
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user