mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-19 16:20:50 +00:00
Merge branch 'master' of github.com:ClickHouse/ClickHouse into variant_inference
This commit is contained in:
avogar
commit
ab45a60d15
3
.github/workflows/master.yml
vendored
3
.github/workflows/master.yml
vendored
@ -104,10 +104,9 @@ jobs:
|
||||
with:
|
||||
stage: Tests_2
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# stage for jobs that do not prohibit merge
|
||||
Tests_3:
|
||||
# Test_3 should not wait for Test_1/Test_2 and should not be blocked by them on master branch since all jobs need to run there.
|
||||
needs: [RunConfig, Builds_1, Builds_2]
|
||||
needs: [RunConfig, Builds_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
|
||||
uses: ./.github/workflows/reusable_test_stage.yml
|
||||
with:
|
||||
|
||||
9
.github/workflows/pull_request.yml
vendored
9
.github/workflows/pull_request.yml
vendored
@ -134,9 +134,8 @@ jobs:
|
||||
with:
|
||||
stage: Tests_2
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# stage for jobs that do not prohibit merge
|
||||
Tests_3:
|
||||
needs: [RunConfig, Builds_1, Tests_1, Builds_2, Tests_2]
|
||||
needs: [RunConfig, Builds_1, Tests_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
|
||||
uses: ./.github/workflows/reusable_test_stage.yml
|
||||
with:
|
||||
@ -157,7 +156,8 @@ jobs:
|
||||
|
||||
CheckReadyForMerge:
|
||||
if: ${{ !cancelled() && needs.StyleCheck.result == 'success' }}
|
||||
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2]
|
||||
# Test_2 or Test_3 must not have jobs required for Mergeable check
|
||||
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1]
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
@ -196,8 +196,7 @@ jobs:
|
||||
concurrency:
|
||||
group: jepsen
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse Keeper Jepsen') }}
|
||||
# jepsen needs binary_release build which is in Builds_2
|
||||
needs: [RunConfig, Builds_2]
|
||||
needs: [RunConfig, Builds_1]
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse Keeper Jepsen
|
||||
|
||||
@ -213,6 +213,7 @@ target_compile_definitions (_poco_foundation
|
||||
)
|
||||
|
||||
target_include_directories (_poco_foundation SYSTEM PUBLIC "include")
|
||||
target_link_libraries (_poco_foundation PRIVATE clickhouse_common_io)
|
||||
|
||||
target_link_libraries (_poco_foundation
|
||||
PRIVATE
|
||||
|
||||
@ -48,7 +48,13 @@ class Foundation_API ThreadPool
|
||||
/// from the pool.
|
||||
{
|
||||
public:
|
||||
ThreadPool(int minCapacity = 2, int maxCapacity = 16, int idleTime = 60, int stackSize = POCO_THREAD_STACK_SIZE);
|
||||
explicit ThreadPool(
|
||||
int minCapacity = 2,
|
||||
int maxCapacity = 16,
|
||||
int idleTime = 60,
|
||||
int stackSize = POCO_THREAD_STACK_SIZE,
|
||||
size_t global_profiler_real_time_period_ns_ = 0,
|
||||
size_t global_profiler_cpu_time_period_ns_ = 0);
|
||||
/// Creates a thread pool with minCapacity threads.
|
||||
/// If required, up to maxCapacity threads are created
|
||||
/// a NoThreadAvailableException exception is thrown.
|
||||
@ -56,8 +62,14 @@ public:
|
||||
/// and more than minCapacity threads are running, the thread
|
||||
/// is killed. Threads are created with given stack size.
|
||||
|
||||
ThreadPool(
|
||||
const std::string & name, int minCapacity = 2, int maxCapacity = 16, int idleTime = 60, int stackSize = POCO_THREAD_STACK_SIZE);
|
||||
explicit ThreadPool(
|
||||
const std::string & name,
|
||||
int minCapacity = 2,
|
||||
int maxCapacity = 16,
|
||||
int idleTime = 60,
|
||||
int stackSize = POCO_THREAD_STACK_SIZE,
|
||||
size_t global_profiler_real_time_period_ns_ = 0,
|
||||
size_t global_profiler_cpu_time_period_ns_ = 0);
|
||||
/// Creates a thread pool with the given name and minCapacity threads.
|
||||
/// If required, up to maxCapacity threads are created
|
||||
/// a NoThreadAvailableException exception is thrown.
|
||||
@ -171,6 +183,8 @@ private:
|
||||
int _serial;
|
||||
int _age;
|
||||
int _stackSize;
|
||||
size_t _globalProfilerRealTimePeriodNs;
|
||||
size_t _globalProfilerCPUTimePeriodNs;
|
||||
ThreadVec _threads;
|
||||
mutable FastMutex _mutex;
|
||||
};
|
||||
|
||||
@ -20,6 +20,7 @@
|
||||
#include "Poco/ErrorHandler.h"
|
||||
#include <sstream>
|
||||
#include <ctime>
|
||||
#include <Common/ThreadPool.h>
|
||||
|
||||
|
||||
namespace Poco {
|
||||
@ -28,7 +29,11 @@ namespace Poco {
|
||||
class PooledThread: public Runnable
|
||||
{
|
||||
public:
|
||||
PooledThread(const std::string& name, int stackSize = POCO_THREAD_STACK_SIZE);
|
||||
explicit PooledThread(
|
||||
const std::string& name,
|
||||
int stackSize = POCO_THREAD_STACK_SIZE,
|
||||
size_t globalProfilerRealTimePeriodNs_ = 0,
|
||||
size_t globalProfilerCPUTimePeriodNs_ = 0);
|
||||
~PooledThread();
|
||||
|
||||
void start();
|
||||
@ -51,16 +56,24 @@ private:
|
||||
Event _targetCompleted;
|
||||
Event _started;
|
||||
FastMutex _mutex;
|
||||
size_t _globalProfilerRealTimePeriodNs;
|
||||
size_t _globalProfilerCPUTimePeriodNs;
|
||||
};
|
||||
|
||||
|
||||
PooledThread::PooledThread(const std::string& name, int stackSize):
|
||||
_idle(true),
|
||||
_idleTime(0),
|
||||
_pTarget(0),
|
||||
_name(name),
|
||||
PooledThread::PooledThread(
|
||||
const std::string& name,
|
||||
int stackSize,
|
||||
size_t globalProfilerRealTimePeriodNs_,
|
||||
size_t globalProfilerCPUTimePeriodNs_) :
|
||||
_idle(true),
|
||||
_idleTime(0),
|
||||
_pTarget(0),
|
||||
_name(name),
|
||||
_thread(name),
|
||||
_targetCompleted(false)
|
||||
_targetCompleted(false),
|
||||
_globalProfilerRealTimePeriodNs(globalProfilerRealTimePeriodNs_),
|
||||
_globalProfilerCPUTimePeriodNs(globalProfilerCPUTimePeriodNs_)
|
||||
{
|
||||
poco_assert_dbg (stackSize >= 0);
|
||||
_thread.setStackSize(stackSize);
|
||||
@ -83,7 +96,7 @@ void PooledThread::start()
|
||||
void PooledThread::start(Thread::Priority priority, Runnable& target)
|
||||
{
|
||||
FastMutex::ScopedLock lock(_mutex);
|
||||
|
||||
|
||||
poco_assert (_pTarget == 0);
|
||||
|
||||
_pTarget = ⌖
|
||||
@ -109,7 +122,7 @@ void PooledThread::start(Thread::Priority priority, Runnable& target, const std:
|
||||
}
|
||||
_thread.setName(fullName);
|
||||
_thread.setPriority(priority);
|
||||
|
||||
|
||||
poco_assert (_pTarget == 0);
|
||||
|
||||
_pTarget = ⌖
|
||||
@ -145,7 +158,7 @@ void PooledThread::join()
|
||||
void PooledThread::activate()
|
||||
{
|
||||
FastMutex::ScopedLock lock(_mutex);
|
||||
|
||||
|
||||
poco_assert (_idle);
|
||||
_idle = false;
|
||||
_targetCompleted.reset();
|
||||
@ -155,7 +168,7 @@ void PooledThread::activate()
|
||||
void PooledThread::release()
|
||||
{
|
||||
const long JOIN_TIMEOUT = 10000;
|
||||
|
||||
|
||||
_mutex.lock();
|
||||
_pTarget = 0;
|
||||
_mutex.unlock();
|
||||
@ -174,6 +187,10 @@ void PooledThread::release()
|
||||
|
||||
void PooledThread::run()
|
||||
{
|
||||
DB::ThreadStatus thread_status;
|
||||
if (unlikely(_globalProfilerRealTimePeriodNs != 0 || _globalProfilerCPUTimePeriodNs != 0))
|
||||
thread_status.initGlobalProfiler(_globalProfilerRealTimePeriodNs, _globalProfilerCPUTimePeriodNs);
|
||||
|
||||
_started.set();
|
||||
for (;;)
|
||||
{
|
||||
@ -220,13 +237,17 @@ void PooledThread::run()
|
||||
ThreadPool::ThreadPool(int minCapacity,
|
||||
int maxCapacity,
|
||||
int idleTime,
|
||||
int stackSize):
|
||||
_minCapacity(minCapacity),
|
||||
_maxCapacity(maxCapacity),
|
||||
int stackSize,
|
||||
size_t globalProfilerRealTimePeriodNs_,
|
||||
size_t globalProfilerCPUTimePeriodNs_) :
|
||||
_minCapacity(minCapacity),
|
||||
_maxCapacity(maxCapacity),
|
||||
_idleTime(idleTime),
|
||||
_serial(0),
|
||||
_age(0),
|
||||
_stackSize(stackSize)
|
||||
_stackSize(stackSize),
|
||||
_globalProfilerRealTimePeriodNs(globalProfilerRealTimePeriodNs_),
|
||||
_globalProfilerCPUTimePeriodNs(globalProfilerCPUTimePeriodNs_)
|
||||
{
|
||||
poco_assert (minCapacity >= 1 && maxCapacity >= minCapacity && idleTime > 0);
|
||||
|
||||
@ -243,14 +264,18 @@ ThreadPool::ThreadPool(const std::string& name,
|
||||
int minCapacity,
|
||||
int maxCapacity,
|
||||
int idleTime,
|
||||
int stackSize):
|
||||
int stackSize,
|
||||
size_t globalProfilerRealTimePeriodNs_,
|
||||
size_t globalProfilerCPUTimePeriodNs_) :
|
||||
_name(name),
|
||||
_minCapacity(minCapacity),
|
||||
_maxCapacity(maxCapacity),
|
||||
_minCapacity(minCapacity),
|
||||
_maxCapacity(maxCapacity),
|
||||
_idleTime(idleTime),
|
||||
_serial(0),
|
||||
_age(0),
|
||||
_stackSize(stackSize)
|
||||
_stackSize(stackSize),
|
||||
_globalProfilerRealTimePeriodNs(globalProfilerRealTimePeriodNs_),
|
||||
_globalProfilerCPUTimePeriodNs(globalProfilerCPUTimePeriodNs_)
|
||||
{
|
||||
poco_assert (minCapacity >= 1 && maxCapacity >= minCapacity && idleTime > 0);
|
||||
|
||||
@ -393,15 +418,15 @@ void ThreadPool::housekeep()
|
||||
ThreadVec activeThreads;
|
||||
idleThreads.reserve(_threads.size());
|
||||
activeThreads.reserve(_threads.size());
|
||||
|
||||
|
||||
for (ThreadVec::iterator it = _threads.begin(); it != _threads.end(); ++it)
|
||||
{
|
||||
if ((*it)->idle())
|
||||
{
|
||||
if ((*it)->idleTime() < _idleTime)
|
||||
idleThreads.push_back(*it);
|
||||
else
|
||||
expiredThreads.push_back(*it);
|
||||
else
|
||||
expiredThreads.push_back(*it);
|
||||
}
|
||||
else activeThreads.push_back(*it);
|
||||
}
|
||||
@ -463,7 +488,7 @@ PooledThread* ThreadPool::createThread()
|
||||
{
|
||||
std::ostringstream name;
|
||||
name << _name << "[#" << ++_serial << "]";
|
||||
return new PooledThread(name.str(), _stackSize);
|
||||
return new PooledThread(name.str(), _stackSize, _globalProfilerRealTimePeriodNs, _globalProfilerCPUTimePeriodNs);
|
||||
}
|
||||
|
||||
|
||||
@ -481,7 +506,7 @@ public:
|
||||
ThreadPool* pool()
|
||||
{
|
||||
FastMutex::ScopedLock lock(_mutex);
|
||||
|
||||
|
||||
if (!_pPool)
|
||||
{
|
||||
_pPool = new ThreadPool("default");
|
||||
@ -490,7 +515,7 @@ public:
|
||||
}
|
||||
return _pPool;
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
ThreadPool* _pPool;
|
||||
FastMutex _mutex;
|
||||
|
||||
@ -26,7 +26,7 @@ namespace Poco
|
||||
{
|
||||
namespace Net
|
||||
{
|
||||
constexpr size_t HTTP_DEFAULT_BUFFER_SIZE = 8 * 1024;
|
||||
constexpr size_t HTTP_DEFAULT_BUFFER_SIZE = 1024 * 1024;
|
||||
|
||||
typedef Poco::BasicBufferedStreamBuf<char, std::char_traits<char>> HTTPBasicStreamBuf;
|
||||
|
||||
|
||||
@ -254,7 +254,7 @@ function run_tests()
|
||||
|
||||
set +e
|
||||
clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
|
||||
--test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
|
||||
--no-drop-if-fail --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
|
||||
| ts '%Y-%m-%d %H:%M:%S' \
|
||||
| tee -a test_output/test_result.txt
|
||||
set -e
|
||||
@ -379,6 +379,10 @@ fi
|
||||
|
||||
tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
|
||||
|
||||
rm -rf /var/lib/clickhouse/data/system/*/
|
||||
tar -chf /test_output/store.tar /var/lib/clickhouse/store ||:
|
||||
tar -chf /test_output/metadata.tar /var/lib/clickhouse/metadata/*.sql ||:
|
||||
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server1.log ||:
|
||||
rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server2.log ||:
|
||||
|
||||
@ -37,7 +37,7 @@ Using named collections:
|
||||
<named_collections>
|
||||
<iceberg_conf>
|
||||
<url>http://test.s3.amazonaws.com/clickhouse-bucket/</url>
|
||||
<access_key_id>test<access_key_id>
|
||||
<access_key_id>test</access_key_id>
|
||||
<secret_access_key>test</secret_access_key>
|
||||
</iceberg_conf>
|
||||
</named_collections>
|
||||
|
||||
@ -13,7 +13,7 @@ This engine provides integration with [Amazon S3](https://aws.amazon.com/s3/) ec
|
||||
CREATE TABLE s3_queue_engine_table (name String, value UInt32)
|
||||
ENGINE = S3Queue(path, [NOSIGN, | aws_access_key_id, aws_secret_access_key,] format, [compression])
|
||||
[SETTINGS]
|
||||
[mode = 'unordered',]
|
||||
[mode = '',]
|
||||
[after_processing = 'keep',]
|
||||
[keeper_path = '',]
|
||||
[s3queue_loading_retries = 0,]
|
||||
|
||||
@ -591,6 +591,22 @@ Default value: 100000
|
||||
<max_part_num_to_warn>400</max_part_num_to_warn>
|
||||
```
|
||||
|
||||
## max\_table\_num\_to\_throw {#max-table-num-to-throw}
|
||||
If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.Default value: 0
|
||||
|
||||
**Example**
|
||||
```xml
|
||||
<max_table_num_to_throw>400</max_table_num_to_throw>
|
||||
```
|
||||
|
||||
## max\_database\_num\_to\_throw {#max-table-num-to-throw}
|
||||
If number of _database is greater than this value, server will throw an exception. 0 means no limitation.
|
||||
Default value: 0
|
||||
|
||||
**Example**
|
||||
```xml
|
||||
<max_database_num_to_throw>400</max_database_num_to_throw>
|
||||
```
|
||||
|
||||
## max_temporary_data_on_disk_size
|
||||
|
||||
|
||||
@ -5418,6 +5418,12 @@ When set to `false` than all attempts are made with identical timeouts.
|
||||
|
||||
Default value: `true`.
|
||||
|
||||
## uniform_snowflake_conversion_functions {#uniform_snowflake_conversion_functions}
|
||||
|
||||
If set to `true`, then functions `snowflakeIDToDateTime`, `snowflakeIDToDateTime64`, `dateTimeToSnowflakeID`, and `dateTime64ToSnowflakeID` are enabled, and functions `snowflakeToDateTime`, `snowflakeToDateTime64`, `dateTimeToSnowflake`, and `dateTime64ToSnowflake` are disabled (and vice versa if set to `false`).
|
||||
|
||||
Default value: `true`
|
||||
|
||||
## allow_experimental_variant_type {#allow_experimental_variant_type}
|
||||
|
||||
Allows creation of experimental [Variant](../../sql-reference/data-types/variant.md).
|
||||
|
||||
@ -113,6 +113,8 @@ Columns:
|
||||
- `used_functions` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `functions`, which were used during query execution.
|
||||
- `used_storages` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `storages`, which were used during query execution.
|
||||
- `used_table_functions` ([Array(String)](../../sql-reference/data-types/array.md)) — Canonical names of `table functions`, which were used during query execution.
|
||||
- `used_privileges` ([Array(String)](../../sql-reference/data-types/array.md)) - Privileges which were successfully checked during query execution.
|
||||
- `missing_privileges` ([Array(String)](../../sql-reference/data-types/array.md)) - Privileges that are missing during query execution.
|
||||
- `query_cache_usage` ([Enum8](../../sql-reference/data-types/enum.md)) — Usage of the [query cache](../query-cache.md) during query execution. Values:
|
||||
- `'Unknown'` = Status unknown.
|
||||
- `'None'` = The query result was neither written into nor read from the query cache.
|
||||
@ -194,6 +196,8 @@ used_formats: []
|
||||
used_functions: []
|
||||
used_storages: []
|
||||
used_table_functions: []
|
||||
used_privileges: []
|
||||
missing_privileges: []
|
||||
query_cache_usage: None
|
||||
```
|
||||
|
||||
|
||||
@ -137,7 +137,7 @@ If the time transition (due to daylight saving time or for other reasons) was pe
|
||||
|
||||
Non-monotonic calendar dates. For example, in Happy Valley - Goose Bay, the time was transitioned one hour backwards at 00:01:00 7 Nov 2010 (one minute after midnight). So after 6th Nov has ended, people observed a whole one minute of 7th Nov, then time was changed back to 23:01 6th Nov and after another 59 minutes the 7th Nov started again. ClickHouse does not (yet) support this kind of fun. During these days the results of time processing functions may be slightly incorrect.
|
||||
|
||||
Similar issue exists for Casey Antarctic station in year 2010. They changed time three hours back at 5 Mar, 02:00. If you are working in antarctic station, please don't afraid to use ClickHouse. Just make sure you set timezone to UTC or be aware of inaccuracies.
|
||||
Similar issue exists for Casey Antarctic station in year 2010. They changed time three hours back at 5 Mar, 02:00. If you are working in antarctic station, please don't be afraid to use ClickHouse. Just make sure you set timezone to UTC or be aware of inaccuracies.
|
||||
|
||||
Time shifts for multiple days. Some pacific islands changed their timezone offset from UTC+14 to UTC-12. That's alright but some inaccuracies may present if you do calculations with their timezone for historical time points at the days of conversion.
|
||||
|
||||
|
||||
@ -2178,6 +2178,32 @@ Result:
|
||||
|
||||
Alias: levenshteinDistance
|
||||
|
||||
## editDistanceUTF8
|
||||
|
||||
Calculates the [edit distance](https://en.wikipedia.org/wiki/Edit_distance) between two UTF8 strings.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
editDistanceUTF8(string1, string2)
|
||||
```
|
||||
|
||||
**Examples**
|
||||
|
||||
``` sql
|
||||
SELECT editDistanceUTF8('我是谁', '我是我');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─editDistanceUTF8('我是谁', '我是我')──┐
|
||||
│ 1 │
|
||||
└─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Alias: levenshteinDistanceUTF8
|
||||
|
||||
## damerauLevenshteinDistance
|
||||
|
||||
Calculates the [Damerau-Levenshtein distance](https://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance) between two byte strings.
|
||||
|
||||
@ -543,12 +543,17 @@ serverUUID()
|
||||
|
||||
Generates a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID).
|
||||
|
||||
The generated Snowflake ID contains the current Unix timestamp in milliseconds 41 (+ 1 top zero bit) bits, followed by machine id (10 bits), a counter (12 bits) to distinguish IDs within a millisecond.
|
||||
The generated Snowflake ID contains the current Unix timestamp in milliseconds (41 + 1 top zero bits), followed by a machine id (10 bits), and a counter (12 bits) to distinguish IDs within a millisecond.
|
||||
For any given timestamp (unix_ts_ms), the counter starts at 0 and is incremented by 1 for each new Snowflake ID until the timestamp changes.
|
||||
In case the counter overflows, the timestamp field is incremented by 1 and the counter is reset to 0.
|
||||
|
||||
Function `generateSnowflakeID` guarantees that the counter field within a timestamp increments monotonically across all function invocations in concurrently running threads and queries.
|
||||
|
||||
:::note
|
||||
The generated Snowflake IDs are based on the UNIX epoch 1970-01-01.
|
||||
While no standard or recommendation exists for the epoch of Snowflake IDs, implementations in other systems may use a different epoch, e.g. Twitter/X (2010-11-04) or Mastodon (2015-01-01).
|
||||
:::
|
||||
|
||||
```
|
||||
0 1 2 3
|
||||
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
|
||||
@ -605,6 +610,11 @@ SELECT generateSnowflakeID(1), generateSnowflakeID(2);
|
||||
|
||||
## snowflakeToDateTime
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Extracts the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) in [DateTime](../data-types/datetime.md) format.
|
||||
|
||||
**Syntax**
|
||||
@ -641,6 +651,11 @@ Result:
|
||||
|
||||
## snowflakeToDateTime64
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Extracts the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) in [DateTime64](../data-types/datetime64.md) format.
|
||||
|
||||
**Syntax**
|
||||
@ -677,6 +692,11 @@ Result:
|
||||
|
||||
## dateTimeToSnowflake
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Converts a [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
@ -711,6 +731,11 @@ Result:
|
||||
|
||||
## dateTime64ToSnowflake
|
||||
|
||||
:::warning
|
||||
This function is deprecated and can only be used if setting [uniform_snowflake_conversion_functions](../../operations/settings/settings.md#uniform_snowflake_conversion_functions) is disabled.
|
||||
The function will be removed at some point in future.
|
||||
:::
|
||||
|
||||
Convert a [DateTime64](../data-types/datetime64.md) to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
@ -743,6 +768,148 @@ Result:
|
||||
└─────────────────────────────┘
|
||||
```
|
||||
|
||||
## snowflakeIDToDateTime
|
||||
|
||||
Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime](../data-types/datetime.md).
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
snowflakeIDToDateTime(value[, epoch[, time_zone]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Snowflake ID. [UInt64](../data-types/int-uint.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
- `time_zone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The timestamp component of `value` as a [DateTime](../data-types/datetime.md) value.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT snowflakeIDToDateTime(7204436857747984384) AS res
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌─────────────────res─┐
|
||||
│ 2024-06-06 10:59:58 │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## snowflakeIDToDateTime64
|
||||
|
||||
Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime64](../data-types/datetime64.md).
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
snowflakeIDToDateTime64(value[, epoch[, time_zone]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Snowflake ID. [UInt64](../data-types/int-uint.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
- `time_zone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The timestamp component of `value` as a [DateTime64](../data-types/datetime64.md) with scale = 3, i.e. millisecond precision.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT snowflakeIDToDateTime64(7204436857747984384) AS res
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌─────────────────res─┐
|
||||
│ 2024-06-06 10:59:58 │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
## dateTimeToSnowflakeID
|
||||
|
||||
Converts a [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
dateTimeToSnowflakeID(value[, epoch])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Date with time. [DateTime](../data-types/datetime.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai') AS dt, dateTimeToSnowflakeID(dt) AS res;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌──────────────────dt─┬─────────────────res─┐
|
||||
│ 2021-08-15 18:57:56 │ 6832626392367104000 │
|
||||
└─────────────────────┴─────────────────────┘
|
||||
```
|
||||
|
||||
## dateTime64ToSnowflakeID
|
||||
|
||||
Convert a [DateTime64](../data-types/datetime64.md) to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
dateTime64ToSnowflakeID(value[, epoch])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Date with time. [DateTime64](../data-types/datetime64.md).
|
||||
- `epoch` - Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT toDateTime('2021-08-15 18:57:56.493', 3, 'Asia/Shanghai') AS dt, dateTime64ToSnowflakeID(dt) AS res;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
┌──────────────────────dt─┬─────────────────res─┐
|
||||
│ 2021-08-15 18:57:56.493 │ 6832626394434895872 │
|
||||
└─────────────────────────┴─────────────────────┘
|
||||
```
|
||||
|
||||
## See also
|
||||
|
||||
- [dictGetUUID](../functions/ext-dict-functions.md#ext_dict_functions-other)
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
#include <Poco/Net/NetException.h>
|
||||
#include <Poco/Util/HelpFormatter.h>
|
||||
#include <Poco/Environment.h>
|
||||
#include <Poco/Config.h>
|
||||
#include <Common/scope_guard_safe.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <base/phdr_cache.h>
|
||||
@ -721,11 +722,6 @@ try
|
||||
CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::getVersionRevision());
|
||||
CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
|
||||
|
||||
Poco::ThreadPool server_pool(3, server_settings.max_connections);
|
||||
std::mutex servers_lock;
|
||||
std::vector<ProtocolServerAdapter> servers;
|
||||
std::vector<ProtocolServerAdapter> servers_to_start_before_tables;
|
||||
|
||||
/** Context contains all that query execution is dependent:
|
||||
* settings, available functions, data types, aggregate functions, databases, ...
|
||||
*/
|
||||
@ -823,6 +819,18 @@ try
|
||||
total_memory_tracker.setSampleMaxAllocationSize(server_settings.total_memory_profiler_sample_max_allocation_size);
|
||||
}
|
||||
|
||||
Poco::ThreadPool server_pool(
|
||||
/* minCapacity */3,
|
||||
/* maxCapacity */server_settings.max_connections,
|
||||
/* idleTime */60,
|
||||
/* stackSize */POCO_THREAD_STACK_SIZE,
|
||||
server_settings.global_profiler_real_time_period_ns,
|
||||
server_settings.global_profiler_cpu_time_period_ns);
|
||||
|
||||
std::mutex servers_lock;
|
||||
std::vector<ProtocolServerAdapter> servers;
|
||||
std::vector<ProtocolServerAdapter> servers_to_start_before_tables;
|
||||
|
||||
/// Wait for all threads to avoid possible use-after-free (for example logging objects can be already destroyed).
|
||||
SCOPE_EXIT({
|
||||
Stopwatch watch;
|
||||
|
||||
@ -4,12 +4,12 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, AccessFlags access_flags_)
|
||||
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccessWrapper> & access_, AccessFlags access_flags_)
|
||||
: CachedAccessChecking(access_, AccessRightsElement{access_flags_})
|
||||
{
|
||||
}
|
||||
|
||||
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, const AccessRightsElement & element_)
|
||||
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccessWrapper> & access_, const AccessRightsElement & element_)
|
||||
: access(access_), element(element_)
|
||||
{
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Access/Common/AccessRightsElement.h>
|
||||
#include <Access/ContextAccess.h>
|
||||
#include <memory>
|
||||
|
||||
|
||||
@ -13,14 +14,14 @@ class ContextAccess;
|
||||
class CachedAccessChecking
|
||||
{
|
||||
public:
|
||||
CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, AccessFlags access_flags_);
|
||||
CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, const AccessRightsElement & element_);
|
||||
CachedAccessChecking(const std::shared_ptr<const ContextAccessWrapper> & access_, AccessFlags access_flags_);
|
||||
CachedAccessChecking(const std::shared_ptr<const ContextAccessWrapper> & access_, const AccessRightsElement & element_);
|
||||
~CachedAccessChecking();
|
||||
|
||||
bool checkAccess(bool throw_if_denied = true);
|
||||
|
||||
private:
|
||||
const std::shared_ptr<const ContextAccess> access;
|
||||
const std::shared_ptr<const ContextAccessWrapper> access;
|
||||
const AccessRightsElement element;
|
||||
bool checked = false;
|
||||
bool result = false;
|
||||
|
||||
@ -20,6 +20,7 @@
|
||||
#include <boost/algorithm/string/join.hpp>
|
||||
#include <boost/range/algorithm/set_algorithm.hpp>
|
||||
#include <cassert>
|
||||
#include <unordered_set>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -271,7 +272,7 @@ namespace
|
||||
|
||||
std::shared_ptr<const ContextAccess> ContextAccess::fromContext(const ContextPtr & context)
|
||||
{
|
||||
return context->getAccess();
|
||||
return ContextAccessWrapper::fromContext(context)->getAccess();
|
||||
}
|
||||
|
||||
|
||||
@ -560,7 +561,7 @@ std::shared_ptr<const AccessRights> ContextAccess::getAccessRightsWithImplicit()
|
||||
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... args) const
|
||||
bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, AccessFlags flags, const Args &... args) const
|
||||
{
|
||||
if (user_was_dropped)
|
||||
{
|
||||
@ -573,8 +574,10 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
if (params.full_access)
|
||||
return true;
|
||||
|
||||
auto access_granted = []
|
||||
auto access_granted = [&]
|
||||
{
|
||||
if constexpr (throw_if_denied)
|
||||
context->addQueryPrivilegesInfo(AccessRightsElement{flags, args...}.toStringWithoutOptions(), true);
|
||||
return true;
|
||||
};
|
||||
|
||||
@ -583,7 +586,10 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
FmtArgs && ...fmt_args [[maybe_unused]])
|
||||
{
|
||||
if constexpr (throw_if_denied)
|
||||
{
|
||||
context->addQueryPrivilegesInfo(AccessRightsElement{flags, args...}.toStringWithoutOptions(), false);
|
||||
throw Exception(error_code, std::move(fmt_string), getUserName(), std::forward<FmtArgs>(fmt_args)...);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
@ -686,102 +692,102 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool ContextAccess::checkAccessImpl(const AccessFlags & flags) const
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessFlags & flags) const
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(flags);
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(context, flags);
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool ContextAccess::checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessFlags & flags, std::string_view database, const Args &... args) const
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(flags, database.empty() ? params.current_database : database, args...);
|
||||
return checkAccessImplHelper<throw_if_denied, grant_option>(context, flags, database.empty() ? params.current_database : database, args...);
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool ContextAccess::checkAccessImplHelper(const AccessRightsElement & element) const
|
||||
bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, const AccessRightsElement & element) const
|
||||
{
|
||||
assert(!element.grant_option || grant_option);
|
||||
if (element.isGlobalWithParameter())
|
||||
{
|
||||
if (element.any_parameter)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags);
|
||||
else
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags, element.parameter);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.parameter);
|
||||
}
|
||||
else if (element.any_database)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags);
|
||||
else if (element.any_table)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags, element.database);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.database);
|
||||
else if (element.any_column)
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags, element.database, element.table);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.database, element.table);
|
||||
else
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(element.access_flags, element.database, element.table, element.columns);
|
||||
return checkAccessImpl<throw_if_denied, grant_option>(context, element.access_flags, element.database, element.table, element.columns);
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool ContextAccess::checkAccessImpl(const AccessRightsElement & element) const
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessRightsElement & element) const
|
||||
{
|
||||
if constexpr (grant_option)
|
||||
{
|
||||
return checkAccessImplHelper<throw_if_denied, true>(element);
|
||||
return checkAccessImplHelper<throw_if_denied, true>(context, element);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (element.grant_option)
|
||||
return checkAccessImplHelper<throw_if_denied, true>(element);
|
||||
return checkAccessImplHelper<throw_if_denied, true>(context, element);
|
||||
else
|
||||
return checkAccessImplHelper<throw_if_denied, false>(element);
|
||||
return checkAccessImplHelper<throw_if_denied, false>(context, element);
|
||||
}
|
||||
}
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const
|
||||
bool ContextAccess::checkAccessImpl(const ContextPtr & context, const AccessRightsElements & elements) const
|
||||
{
|
||||
for (const auto & element : elements)
|
||||
if (!checkAccessImpl<throw_if_denied, grant_option>(element))
|
||||
if (!checkAccessImpl<throw_if_denied, grant_option>(context, element))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl<false, false>(flags); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false>(flags, database); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false>(flags, database, table); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false>(flags, database, table, column); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false>(flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const AccessRightsElement & element) const { return checkAccessImpl<false, false>(element); }
|
||||
bool ContextAccess::isGranted(const AccessRightsElements & elements) const { return checkAccessImpl<false, false>(elements); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags) const { return checkAccessImpl<false, false>(context, flags); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, false>(context, flags, database); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, false>(context, flags, database, table); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, false>(context, flags, database, table, column); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, false>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, false>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessRightsElement & element) const { return checkAccessImpl<false, false>(context, element); }
|
||||
bool ContextAccess::isGranted(const ContextPtr & context, const AccessRightsElements & elements) const { return checkAccessImpl<false, false>(context, elements); }
|
||||
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags) const { return checkAccessImpl<false, true>(flags); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true>(flags, database); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true>(flags, database, table); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true>(flags, database, table, column); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true>(flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const AccessRightsElement & element) const { return checkAccessImpl<false, true>(element); }
|
||||
bool ContextAccess::hasGrantOption(const AccessRightsElements & elements) const { return checkAccessImpl<false, true>(elements); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags) const { return checkAccessImpl<false, true>(context, flags); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { return checkAccessImpl<false, true>(context, flags, database); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { return checkAccessImpl<false, true>(context, flags, database, table); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return checkAccessImpl<false, true>(context, flags, database, table, column); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return checkAccessImpl<false, true>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return checkAccessImpl<false, true>(context, flags, database, table, columns); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessRightsElement & element) const { return checkAccessImpl<false, true>(context, element); }
|
||||
bool ContextAccess::hasGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const { return checkAccessImpl<false, true>(context, elements); }
|
||||
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags) const { checkAccessImpl<true, false>(flags); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false>(flags, database); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false>(flags, database, table); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false>(flags, database, table, column); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false>(flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const AccessRightsElement & element) const { checkAccessImpl<true, false>(element); }
|
||||
void ContextAccess::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl<true, false>(elements); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags) const { checkAccessImpl<true, false>(context, flags); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, false>(context, flags, database); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, false>(context, flags, database, table); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, false>(context, flags, database, table, column); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, false>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, false>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessRightsElement & element) const { checkAccessImpl<true, false>(context, element); }
|
||||
void ContextAccess::checkAccess(const ContextPtr & context, const AccessRightsElements & elements) const { checkAccessImpl<true, false>(context, elements); }
|
||||
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags) const { checkAccessImpl<true, true>(flags); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true>(flags, database); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true>(flags, database, table); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true>(flags, database, table, column); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true>(flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const AccessRightsElement & element) const { checkAccessImpl<true, true>(element); }
|
||||
void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl<true, true>(elements); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags) const { checkAccessImpl<true, true>(context, flags); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const { checkAccessImpl<true, true>(context, flags, database); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const { checkAccessImpl<true, true>(context, flags, database, table); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { checkAccessImpl<true, true>(context, flags, database, table, column); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { checkAccessImpl<true, true>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { checkAccessImpl<true, true>(context, flags, database, table, columns); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessRightsElement & element) const { checkAccessImpl<true, true>(context, element); }
|
||||
void ContextAccess::checkGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const { checkAccessImpl<true, true>(context, elements); }
|
||||
|
||||
|
||||
template <bool throw_if_denied, typename Container, typename GetNameFunction>
|
||||
bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const GetNameFunction & get_name_function) const
|
||||
bool ContextAccess::checkAdminOptionImplHelper(const ContextPtr & context, const Container & role_ids, const GetNameFunction & get_name_function) const
|
||||
{
|
||||
auto show_error = []<typename... FmtArgs>(int error_code [[maybe_unused]],
|
||||
FormatStringHelper<FmtArgs...> fmt_string [[maybe_unused]],
|
||||
@ -804,7 +810,7 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const
|
||||
if (!std::size(role_ids))
|
||||
return true;
|
||||
|
||||
if (isGranted(AccessType::ROLE_ADMIN))
|
||||
if (isGranted(context, AccessType::ROLE_ADMIN))
|
||||
return true;
|
||||
|
||||
auto info = getRolesInfo();
|
||||
@ -840,54 +846,54 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const UUID & role_id) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(to_array(role_id), [this](const UUID & id, size_t) { return access_control->tryReadName(id); });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, to_array(role_id), [this](const UUID & id, size_t) { return access_control->tryReadName(id); });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const UUID & role_id, const String & role_name) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id, const String & role_name) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(to_array(role_id), [&role_name](const UUID &, size_t) { return std::optional<String>{role_name}; });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, to_array(role_id), [&role_name](const UUID &, size_t) { return std::optional<String>{role_name}; });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(to_array(role_id), [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, to_array(role_id), [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const std::vector<UUID> & role_ids) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(role_ids, [this](const UUID & id, size_t) { return access_control->tryReadName(id); });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, role_ids, [this](const UUID & id, size_t) { return access_control->tryReadName(id); });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(role_ids, [&names_of_roles](const UUID &, size_t i) { return std::optional<String>{names_of_roles[i]}; });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, role_ids, [&names_of_roles](const UUID &, size_t i) { return std::optional<String>{names_of_roles[i]}; });
|
||||
}
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool ContextAccess::checkAdminOptionImpl(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
bool ContextAccess::checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const
|
||||
{
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(role_ids, [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
return checkAdminOptionImplHelper<throw_if_denied>(context, role_ids, [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional<String>{}; });
|
||||
}
|
||||
|
||||
bool ContextAccess::hasAdminOption(const UUID & role_id) const { return checkAdminOptionImpl<false>(role_id); }
|
||||
bool ContextAccess::hasAdminOption(const UUID & role_id, const String & role_name) const { return checkAdminOptionImpl<false>(role_id, role_name); }
|
||||
bool ContextAccess::hasAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { return checkAdminOptionImpl<false>(role_id, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const std::vector<UUID> & role_ids) const { return checkAdminOptionImpl<false>(role_ids); }
|
||||
bool ContextAccess::hasAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { return checkAdminOptionImpl<false>(role_ids, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { return checkAdminOptionImpl<false>(role_ids, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const UUID & role_id) const { return checkAdminOptionImpl<false>(context, role_id); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const UUID & role_id, const String & role_name) const { return checkAdminOptionImpl<false>(context, role_id, role_name); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { return checkAdminOptionImpl<false>(context, role_id, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids) const { return checkAdminOptionImpl<false>(context, role_ids); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { return checkAdminOptionImpl<false>(context, role_ids, names_of_roles); }
|
||||
bool ContextAccess::hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { return checkAdminOptionImpl<false>(context, role_ids, names_of_roles); }
|
||||
|
||||
void ContextAccess::checkAdminOption(const UUID & role_id) const { checkAdminOptionImpl<true>(role_id); }
|
||||
void ContextAccess::checkAdminOption(const UUID & role_id, const String & role_name) const { checkAdminOptionImpl<true>(role_id, role_name); }
|
||||
void ContextAccess::checkAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { checkAdminOptionImpl<true>(role_id, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids) const { checkAdminOptionImpl<true>(role_ids); }
|
||||
void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { checkAdminOptionImpl<true>(role_ids, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { checkAdminOptionImpl<true>(role_ids, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const UUID & role_id) const { checkAdminOptionImpl<true>(context, role_id); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const UUID & role_id, const String & role_name) const { checkAdminOptionImpl<true>(context, role_id, role_name); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { checkAdminOptionImpl<true>(context, role_id, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids) const { checkAdminOptionImpl<true>(context, role_ids); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { checkAdminOptionImpl<true>(context, role_ids, names_of_roles); }
|
||||
void ContextAccess::checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { checkAdminOptionImpl<true>(context, role_ids, names_of_roles); }
|
||||
|
||||
|
||||
void ContextAccess::checkGranteeIsAllowed(const UUID & grantee_id, const IAccessEntity & grantee) const
|
||||
@ -919,4 +925,10 @@ void ContextAccess::checkGranteesAreAllowed(const std::vector<UUID> & grantee_id
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<const ContextAccessWrapper> ContextAccessWrapper::fromContext(const ContextPtr & context)
|
||||
{
|
||||
return context->getAccess();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -4,9 +4,12 @@
|
||||
#include <Access/ContextAccessParams.h>
|
||||
#include <Access/EnabledRowPolicies.h>
|
||||
#include <Interpreters/ClientInfo.h>
|
||||
#include <Access/QuotaUsage.h>
|
||||
#include <Common/SettingsChanges.h>
|
||||
#include <Core/UUID.h>
|
||||
#include <base/scope_guard.h>
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
@ -71,59 +74,59 @@ public:
|
||||
|
||||
/// Checks if a specified access is granted, and throws an exception if not.
|
||||
/// Empty database means the current database.
|
||||
void checkAccess(const AccessFlags & flags) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkAccess(const AccessRightsElement & element) const;
|
||||
void checkAccess(const AccessRightsElements & elements) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
void checkAccess(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
void checkGrantOption(const AccessFlags & flags) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkGrantOption(const AccessRightsElement & element) const;
|
||||
void checkGrantOption(const AccessRightsElements & elements) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
void checkGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
/// Checks if a specified access is granted, and returns false if not.
|
||||
/// Empty database means the current database.
|
||||
bool isGranted(const AccessFlags & flags) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool isGranted(const AccessRightsElement & element) const;
|
||||
bool isGranted(const AccessRightsElements & elements) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
bool isGranted(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
bool hasGrantOption(const AccessFlags & flags) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool hasGrantOption(const AccessRightsElement & element) const;
|
||||
bool hasGrantOption(const AccessRightsElements & elements) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
bool hasGrantOption(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
/// Checks if a specified role is granted with admin option, and throws an exception if not.
|
||||
void checkAdminOption(const UUID & role_id) const;
|
||||
void checkAdminOption(const UUID & role_id, const String & role_name) const;
|
||||
void checkAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
void checkAdminOption(const std::vector<UUID> & role_ids) const;
|
||||
void checkAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
void checkAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
void checkAdminOption(const ContextPtr & context, const UUID & role_id) const;
|
||||
void checkAdminOption(const ContextPtr & context, const UUID & role_id, const String & role_name) const;
|
||||
void checkAdminOption(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
void checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids) const;
|
||||
void checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
void checkAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
|
||||
/// Checks if a specified role is granted with admin option, and returns false if not.
|
||||
bool hasAdminOption(const UUID & role_id) const;
|
||||
bool hasAdminOption(const UUID & role_id, const String & role_name) const;
|
||||
bool hasAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool hasAdminOption(const std::vector<UUID> & role_ids) const;
|
||||
bool hasAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
bool hasAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const UUID & role_id) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const UUID & role_id, const String & role_name) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
bool hasAdminOption(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
|
||||
/// Checks if a grantee is allowed for the current user, throws an exception if not.
|
||||
void checkGranteeIsAllowed(const UUID & grantee_id, const IAccessEntity & grantee) const;
|
||||
@ -142,43 +145,43 @@ private:
|
||||
void calculateAccessRights() const TSA_REQUIRES(mutex);
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImpl(const AccessFlags & flags) const;
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessFlags & flags) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool checkAccessImpl(const AccessFlags & flags, std::string_view database, const Args &... args) const;
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessFlags & flags, std::string_view database, const Args &... args) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImpl(const AccessRightsElement & element) const;
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImpl(const AccessRightsElements & elements) const;
|
||||
bool checkAccessImpl(const ContextPtr & context, const AccessRightsElements & elements) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option, typename... Args>
|
||||
bool checkAccessImplHelper(AccessFlags flags, const Args &... args) const;
|
||||
bool checkAccessImplHelper(const ContextPtr & context, AccessFlags flags, const Args &... args) const;
|
||||
|
||||
template <bool throw_if_denied, bool grant_option>
|
||||
bool checkAccessImplHelper(const AccessRightsElement & element) const;
|
||||
bool checkAccessImplHelper(const ContextPtr & context, const AccessRightsElement & element) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const UUID & role_id) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const UUID & role_id, const String & role_name) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id, const String & role_name) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const std::vector<UUID> & role_ids) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids, const Strings & names_of_roles) const;
|
||||
|
||||
template <bool throw_if_denied>
|
||||
bool checkAdminOptionImpl(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
bool checkAdminOptionImpl(const ContextPtr & context, const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const;
|
||||
|
||||
template <bool throw_if_denied, typename Container, typename GetNameFunction>
|
||||
bool checkAdminOptionImplHelper(const Container & role_ids, const GetNameFunction & get_name_function) const;
|
||||
bool checkAdminOptionImplHelper(const ContextPtr & context, const Container & role_ids, const GetNameFunction & get_name_function) const;
|
||||
|
||||
const AccessControl * access_control = nullptr;
|
||||
const Params params;
|
||||
@ -203,4 +206,115 @@ private:
|
||||
mutable std::shared_ptr<const EnabledSettings> enabled_settings TSA_GUARDED_BY(mutex);
|
||||
};
|
||||
|
||||
/// This wrapper was added to be able to pass the current context to the access
|
||||
/// without the need to change the signature and all calls to the ContextAccess itself.
|
||||
/// Right now a context is used to store privileges that are checked for a query,
|
||||
/// and might be useful for something else in the future as well.
|
||||
class ContextAccessWrapper : public std::enable_shared_from_this<ContextAccessWrapper>
|
||||
{
|
||||
public:
|
||||
using ContextAccessPtr = std::shared_ptr<const ContextAccess>;
|
||||
|
||||
ContextAccessWrapper(const ContextAccessPtr & access_, const ContextPtr & context_): access(access_), context(context_) {}
|
||||
~ContextAccessWrapper() = default;
|
||||
|
||||
static std::shared_ptr<const ContextAccessWrapper> fromContext(const ContextPtr & context);
|
||||
|
||||
const ContextAccess::Params & getParams() const { return access->getParams(); }
|
||||
|
||||
const ContextAccessPtr & getAccess() const { return access; }
|
||||
|
||||
/// Returns the current user. Throws if user is nullptr.
|
||||
ALWAYS_INLINE UserPtr getUser() const { return access->getUser(); }
|
||||
/// Same as above, but can return nullptr.
|
||||
ALWAYS_INLINE UserPtr tryGetUser() const { return access->tryGetUser(); }
|
||||
ALWAYS_INLINE String getUserName() const { return access->getUserName(); }
|
||||
ALWAYS_INLINE std::optional<UUID> getUserID() const { return access->getUserID(); }
|
||||
|
||||
/// Returns information about current and enabled roles.
|
||||
ALWAYS_INLINE std::shared_ptr<const EnabledRolesInfo> getRolesInfo() const { return access->getRolesInfo(); }
|
||||
|
||||
/// Returns the row policy filter for a specified table.
|
||||
/// The function returns nullptr if there is no filter to apply.
|
||||
ALWAYS_INLINE RowPolicyFilterPtr getRowPolicyFilter(const String & database, const String & table_name, RowPolicyFilterType filter_type) const { return access->getRowPolicyFilter(database, table_name, filter_type); }
|
||||
|
||||
/// Returns the quota to track resource consumption.
|
||||
ALWAYS_INLINE std::shared_ptr<const EnabledQuota> getQuota() const { return access->getQuota(); }
|
||||
ALWAYS_INLINE std::optional<QuotaUsage> getQuotaUsage() const { return access->getQuotaUsage(); }
|
||||
|
||||
/// Returns the default settings, i.e. the settings which should be applied on user's login.
|
||||
ALWAYS_INLINE SettingsChanges getDefaultSettings() const { return access->getDefaultSettings(); }
|
||||
ALWAYS_INLINE std::shared_ptr<const SettingsProfilesInfo> getDefaultProfileInfo() const { return access->getDefaultProfileInfo(); }
|
||||
|
||||
/// Returns the current access rights.
|
||||
ALWAYS_INLINE std::shared_ptr<const AccessRights> getAccessRights() const { return access->getAccessRights(); }
|
||||
ALWAYS_INLINE std::shared_ptr<const AccessRights> getAccessRightsWithImplicit() const { return access->getAccessRightsWithImplicit(); }
|
||||
|
||||
/// Checks if a specified access is granted, and throws an exception if not.
|
||||
/// Empty database means the current database.
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags) const { access->checkAccess(context, flags); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database) const { access->checkAccess(context, flags, database); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table) const { access->checkAccess(context, flags, database, table); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { access->checkAccess(context, flags, database, table, column); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { access->checkAccess(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { access->checkAccess(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessRightsElement & element) const { access->checkAccess(context, element); }
|
||||
ALWAYS_INLINE void checkAccess(const AccessRightsElements & elements) const { access->checkAccess(context, elements); }
|
||||
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags) const { access->checkGrantOption(context, flags); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database) const { access->checkGrantOption(context, flags, database); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { access->checkGrantOption(context, flags, database, table); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { access->checkGrantOption(context, flags, database, table, column); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { access->checkGrantOption(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { access->checkGrantOption(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessRightsElement & element) const { access->checkGrantOption(context, element); }
|
||||
ALWAYS_INLINE void checkGrantOption(const AccessRightsElements & elements) const { access->checkGrantOption(context, elements); }
|
||||
|
||||
/// Checks if a specified access is granted, and returns false if not.
|
||||
/// Empty database means the current database.
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags) const { return access->isGranted(context, flags); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database) const { return access->isGranted(context, flags, database); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table) const { return access->isGranted(context, flags, database, table); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return access->isGranted(context, flags, database, table, column); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return access->isGranted(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return access->isGranted(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessRightsElement & element) const { return access->isGranted(context, element); }
|
||||
ALWAYS_INLINE bool isGranted(const AccessRightsElements & elements) const { return access->isGranted(context, elements); }
|
||||
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags) const { return access->hasGrantOption(context, flags); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database) const { return access->hasGrantOption(context, flags, database); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table) const { return access->hasGrantOption(context, flags, database, table); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, std::string_view column) const { return access->hasGrantOption(context, flags, database, table, column); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const std::vector<std::string_view> & columns) const { return access->hasGrantOption(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessFlags & flags, std::string_view database, std::string_view table, const Strings & columns) const { return access->hasGrantOption(context, flags, database, table, columns); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessRightsElement & element) const { return access->hasGrantOption(context, element); }
|
||||
ALWAYS_INLINE bool hasGrantOption(const AccessRightsElements & elements) const { return access->hasGrantOption(context, elements); }
|
||||
|
||||
/// Checks if a specified role is granted with admin option, and throws an exception if not.
|
||||
ALWAYS_INLINE void checkAdminOption(const UUID & role_id) const { access->checkAdminOption(context, role_id); }
|
||||
ALWAYS_INLINE void checkAdminOption(const UUID & role_id, const String & role_name) const { access->checkAdminOption(context, role_id, role_name); }
|
||||
ALWAYS_INLINE void checkAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { access->checkAdminOption(context, role_id, names_of_roles); }
|
||||
ALWAYS_INLINE void checkAdminOption(const std::vector<UUID> & role_ids) const { access->checkAdminOption(context, role_ids); }
|
||||
ALWAYS_INLINE void checkAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { access->checkAdminOption(context, role_ids, names_of_roles); }
|
||||
ALWAYS_INLINE void checkAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { access->checkAdminOption(context, role_ids, names_of_roles); }
|
||||
|
||||
/// Checks if a specified role is granted with admin option, and returns false if not.
|
||||
ALWAYS_INLINE bool hasAdminOption(const UUID & role_id) const { return access->hasAdminOption(context, role_id); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const UUID & role_id, const String & role_name) const { return access->hasAdminOption(context, role_id, role_name); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const UUID & role_id, const std::unordered_map<UUID, String> & names_of_roles) const { return access->hasAdminOption(context, role_id, names_of_roles); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const std::vector<UUID> & role_ids) const { return access->hasAdminOption(context, role_ids); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const std::vector<UUID> & role_ids, const Strings & names_of_roles) const { return access->hasAdminOption(context, role_ids, names_of_roles); }
|
||||
ALWAYS_INLINE bool hasAdminOption(const std::vector<UUID> & role_ids, const std::unordered_map<UUID, String> & names_of_roles) const { return access->hasAdminOption(context, role_ids, names_of_roles); }
|
||||
|
||||
/// Checks if a grantee is allowed for the current user, throws an exception if not.
|
||||
ALWAYS_INLINE void checkGranteeIsAllowed(const UUID & grantee_id, const IAccessEntity & grantee) const { access->checkGranteeIsAllowed(grantee_id, grantee); }
|
||||
/// Checks if grantees are allowed for the current user, throws an exception if not.
|
||||
ALWAYS_INLINE void checkGranteesAreAllowed(const std::vector<UUID> & grantee_ids) const { access->checkGranteesAreAllowed(grantee_ids); }
|
||||
|
||||
private:
|
||||
ContextAccessPtr access;
|
||||
ContextPtr context;
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -15,22 +15,8 @@ namespace ErrorCodes
|
||||
|
||||
bool operator==(const SettingsProfilesInfo & lhs, const SettingsProfilesInfo & rhs)
|
||||
{
|
||||
if (lhs.settings != rhs.settings)
|
||||
return false;
|
||||
|
||||
if (lhs.constraints != rhs.constraints)
|
||||
return false;
|
||||
|
||||
if (lhs.profiles != rhs.profiles)
|
||||
return false;
|
||||
|
||||
if (lhs.profiles_with_implicit != rhs.profiles_with_implicit)
|
||||
return false;
|
||||
|
||||
if (lhs.names_of_profiles != rhs.names_of_profiles)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
return std::tie(lhs.settings, lhs.constraints, lhs.profiles, lhs.profiles_with_implicit, lhs.names_of_profiles)
|
||||
== std::tie(rhs.settings, rhs.constraints, rhs.profiles, rhs.profiles_with_implicit, rhs.names_of_profiles);
|
||||
}
|
||||
|
||||
std::shared_ptr<const SettingsConstraintsAndProfileIDs>
|
||||
@ -66,18 +52,20 @@ Strings SettingsProfilesInfo::getProfileNames() const
|
||||
{
|
||||
Strings result;
|
||||
result.reserve(profiles.size());
|
||||
for (const auto & profile_id : profiles)
|
||||
for (const UUID & profile_uuid : profiles)
|
||||
{
|
||||
const auto p = names_of_profiles.find(profile_id);
|
||||
if (p != names_of_profiles.end())
|
||||
result.push_back(p->second);
|
||||
const auto names_it = names_of_profiles.find(profile_uuid);
|
||||
if (names_it != names_of_profiles.end())
|
||||
{
|
||||
result.push_back(names_it->second);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (const auto name = access_control.tryReadName(profile_id))
|
||||
if (const auto name = access_control.tryReadName(profile_uuid))
|
||||
// We could've updated cache here, but it is a very rare case, so don't bother.
|
||||
result.push_back(*name);
|
||||
else
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unable to get profile name for {}", toString(profile_id));
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unable to get profile name for {}", toString(profile_uuid));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -29,7 +29,11 @@ struct SettingsProfilesInfo
|
||||
/// Names of all the profiles in `profiles`.
|
||||
std::unordered_map<UUID, String> names_of_profiles;
|
||||
|
||||
explicit SettingsProfilesInfo(const AccessControl & access_control_) : constraints(access_control_), access_control(access_control_) {}
|
||||
explicit SettingsProfilesInfo(const AccessControl & access_control_)
|
||||
: constraints(access_control_), access_control(access_control_)
|
||||
{
|
||||
}
|
||||
|
||||
std::shared_ptr<const SettingsConstraintsAndProfileIDs> getConstraintsAndProfileIDs(
|
||||
const std::shared_ptr<const SettingsConstraintsAndProfileIDs> & previous = nullptr) const;
|
||||
|
||||
|
||||
@ -228,6 +228,11 @@ public:
|
||||
return prefix_size + nested_func->sizeOfData();
|
||||
}
|
||||
|
||||
size_t alignOfData() const override
|
||||
{
|
||||
return std::max(alignof(Data), nested_func->alignOfData());
|
||||
}
|
||||
|
||||
void create(AggregateDataPtr __restrict place) const override
|
||||
{
|
||||
new (place) Data;
|
||||
|
||||
@ -9,6 +9,8 @@
|
||||
#include <Interpreters/convertFieldToType.h>
|
||||
#include <Interpreters/Set.h>
|
||||
|
||||
#include <Common/assert_cast.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -54,8 +56,9 @@ size_t getCompoundTypeDepth(const IDataType & type)
|
||||
}
|
||||
|
||||
template <typename Collection>
|
||||
Block createBlockFromCollection(const Collection & collection, const DataTypes & block_types, bool transform_null_in)
|
||||
Block createBlockFromCollection(const Collection & collection, const DataTypes& value_types, const DataTypes & block_types, bool transform_null_in)
|
||||
{
|
||||
assert(collection.size() == value_types.size());
|
||||
size_t columns_size = block_types.size();
|
||||
MutableColumns columns(columns_size);
|
||||
for (size_t i = 0; i < columns_size; ++i)
|
||||
@ -66,13 +69,17 @@ Block createBlockFromCollection(const Collection & collection, const DataTypes &
|
||||
|
||||
Row tuple_values;
|
||||
|
||||
for (const auto & value : collection)
|
||||
for (size_t collection_index = 0; collection_index < collection.size(); ++collection_index)
|
||||
{
|
||||
const auto & value = collection[collection_index];
|
||||
if (columns_size == 1)
|
||||
{
|
||||
auto field = convertFieldToTypeStrict(value, *block_types[0]);
|
||||
const DataTypePtr & data_type = value_types[collection_index];
|
||||
auto field = convertFieldToTypeStrict(value, *data_type, *block_types[0]);
|
||||
if (!field)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
bool need_insert_null = transform_null_in && block_types[0]->isNullable();
|
||||
if (!field->isNull() || need_insert_null)
|
||||
@ -87,6 +94,9 @@ Block createBlockFromCollection(const Collection & collection, const DataTypes &
|
||||
value.getTypeName());
|
||||
|
||||
const auto & tuple = value.template get<const Tuple &>();
|
||||
const DataTypePtr & value_type = value_types[collection_index];
|
||||
const DataTypes & tuple_value_type = typeid_cast<const DataTypeTuple *>(value_type.get())->getElements();
|
||||
|
||||
size_t tuple_size = tuple.size();
|
||||
|
||||
if (tuple_size != columns_size)
|
||||
@ -101,7 +111,7 @@ Block createBlockFromCollection(const Collection & collection, const DataTypes &
|
||||
size_t i = 0;
|
||||
for (; i < tuple_size; ++i)
|
||||
{
|
||||
auto converted_field = convertFieldToTypeStrict(tuple[i], *block_types[i]);
|
||||
auto converted_field = convertFieldToTypeStrict(tuple[i], *tuple_value_type[i], *block_types[i]);
|
||||
if (!converted_field)
|
||||
break;
|
||||
tuple_values[i] = std::move(*converted_field);
|
||||
@ -147,20 +157,28 @@ Block getSetElementsForConstantValue(const DataTypePtr & expression_type, const
|
||||
if (lhs_type_depth == rhs_type_depth)
|
||||
{
|
||||
/// 1 in 1; (1, 2) in (1, 2); identity(tuple(tuple(tuple(1)))) in tuple(tuple(tuple(1))); etc.
|
||||
|
||||
Array array{value};
|
||||
result_block = createBlockFromCollection(array, set_element_types, transform_null_in);
|
||||
DataTypes value_types{value_type};
|
||||
result_block = createBlockFromCollection(array, value_types, set_element_types, transform_null_in);
|
||||
}
|
||||
else if (lhs_type_depth + 1 == rhs_type_depth)
|
||||
{
|
||||
/// 1 in (1, 2); (1, 2) in ((1, 2), (3, 4))
|
||||
|
||||
WhichDataType rhs_which_type(value_type);
|
||||
|
||||
if (rhs_which_type.isArray())
|
||||
result_block = createBlockFromCollection(value.get<const Array &>(), set_element_types, transform_null_in);
|
||||
{
|
||||
const DataTypeArray * value_array_type = assert_cast<const DataTypeArray *>(value_type.get());
|
||||
size_t value_array_size = value.get<const Array &>().size();
|
||||
DataTypes value_types(value_array_size, value_array_type->getNestedType());
|
||||
result_block = createBlockFromCollection(value.get<const Array &>(), value_types, set_element_types, transform_null_in);
|
||||
}
|
||||
else if (rhs_which_type.isTuple())
|
||||
result_block = createBlockFromCollection(value.get<const Tuple &>(), set_element_types, transform_null_in);
|
||||
{
|
||||
const DataTypeTuple * value_tuple_type = assert_cast<const DataTypeTuple *>(value_type.get());
|
||||
const DataTypes & value_types = value_tuple_type->getElements();
|
||||
result_block = createBlockFromCollection(value.get<const Tuple &>(), value_types, set_element_types, transform_null_in);
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Unsupported type at the right-side of IN. Expected Array or Tuple. Actual {}",
|
||||
|
||||
@ -255,6 +255,17 @@ void HedgedConnections::sendCancel()
|
||||
if (!sent_query || cancelled)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot cancel. Either no query sent or already cancelled.");
|
||||
|
||||
/// All hedged connections should be stopped, since otherwise before the
|
||||
/// HedgedConnectionsFactory will be destroyed (that will happen from
|
||||
/// QueryPipeline dtor) they could still do some work.
|
||||
/// And not only this does not make sense, but it also could lead to
|
||||
/// use-after-free of the current_thread, since the thread from which they
|
||||
/// had been created differs from the thread where the dtor of
|
||||
/// QueryPipeline will be called and the initial thread could be already
|
||||
/// destroyed (especially when the system is under pressure).
|
||||
if (hedged_connections_factory.hasEventsInProcess())
|
||||
hedged_connections_factory.stopChoosingReplicas();
|
||||
|
||||
cancelled = true;
|
||||
|
||||
for (auto & offset_status : offset_states)
|
||||
|
||||
@ -602,6 +602,8 @@
|
||||
M(721, DEPRECATED_FUNCTION) \
|
||||
M(722, ASYNC_LOAD_WAIT_FAILED) \
|
||||
M(723, PARQUET_EXCEPTION) \
|
||||
M(724, TOO_MANY_TABLES) \
|
||||
M(725, TOO_MANY_DATABASES) \
|
||||
\
|
||||
M(900, DISTRIBUTED_CACHE_ERROR) \
|
||||
M(901, CANNOT_USE_DISTRIBUTED_CACHE) \
|
||||
|
||||
@ -406,13 +406,19 @@ private:
|
||||
void processQueue(std::unique_lock<std::mutex> && lock)
|
||||
{
|
||||
if (events.empty())
|
||||
return processActivation(std::move(lock));
|
||||
{
|
||||
processActivation(std::move(lock));
|
||||
return;
|
||||
}
|
||||
if (activations.empty())
|
||||
return processEvent(std::move(lock));
|
||||
{
|
||||
processEvent(std::move(lock));
|
||||
return;
|
||||
}
|
||||
if (activations.front().activation_event_id < events.front().event_id)
|
||||
return processActivation(std::move(lock));
|
||||
processActivation(std::move(lock));
|
||||
else
|
||||
return processEvent(std::move(lock));
|
||||
processEvent(std::move(lock));
|
||||
}
|
||||
|
||||
void processActivation(std::unique_lock<std::mutex> && lock)
|
||||
|
||||
@ -102,6 +102,8 @@ namespace DB
|
||||
M(UInt64, max_dictionary_num_to_warn, 1000lu, "If the number of dictionaries is greater than this value, the server will create a warning that will displayed to user.", 0) \
|
||||
M(UInt64, max_database_num_to_warn, 1000lu, "If the number of databases is greater than this value, the server will create a warning that will displayed to user.", 0) \
|
||||
M(UInt64, max_part_num_to_warn, 100000lu, "If the number of parts is greater than this value, the server will create a warning that will displayed to user.", 0) \
|
||||
M(UInt64, max_table_num_to_throw, 0lu, "If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \
|
||||
M(UInt64, max_database_num_to_throw, 0lu, "If number of databases is greater than this value, server will throw an exception. 0 means no limitation.", 0) \
|
||||
M(UInt64, concurrent_threads_soft_limit_num, 0, "Sets how many concurrent thread can be allocated before applying CPU pressure. Zero means unlimited.", 0) \
|
||||
M(UInt64, concurrent_threads_soft_limit_ratio_to_cores, 0, "Same as concurrent_threads_soft_limit_num, but with ratio to cores.", 0) \
|
||||
\
|
||||
|
||||
@ -933,6 +933,7 @@ class IColumn;
|
||||
M(Int64, prefer_warmed_unmerged_parts_seconds, 0, "Only available in ClickHouse Cloud. If a merged part is less than this many seconds old and is not pre-warmed (see cache_populated_by_fetch), but all its source parts are available and pre-warmed, SELECT queries will read from those parts instead. Only for ReplicatedMergeTree. Note that this only checks whether CacheWarmer processed the part; if the part was fetched into cache by something else, it'll still be considered cold until CacheWarmer gets to it; if it was warmed, then evicted from cache, it'll still be considered warm.", 0) \
|
||||
M(Bool, iceberg_engine_ignore_schema_evolution, false, "Ignore schema evolution in Iceberg table engine and read all data using latest schema saved on table creation. Note that it can lead to incorrect result", 0) \
|
||||
M(Bool, allow_deprecated_error_prone_window_functions, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)", 0) \
|
||||
M(Bool, uniform_snowflake_conversion_functions, true, "Enables functions snowflakeIDToDateTime[64] and dateTime[64]ToSnowflakeID while disabling functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake.", 0) \
|
||||
|
||||
// End of COMMON_SETTINGS
|
||||
// Please add settings related to formats into the FORMAT_FACTORY_SETTINGS, move obsolete settings to OBSOLETE_SETTINGS and obsolete format settings to OBSOLETE_FORMAT_SETTINGS.
|
||||
|
||||
@ -102,6 +102,7 @@ static const std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges
|
||||
{"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."},
|
||||
{"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."},
|
||||
{"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"},
|
||||
{"uniform_snowflake_conversion_functions", false, true, "Enable functions snowflakeIDToDateTime[64] and dateTime[64]ToSnowflakeID."},
|
||||
{"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."},
|
||||
{"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."},
|
||||
{"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."},
|
||||
|
||||
@ -186,6 +186,7 @@ void DatabaseLazy::attachTable(ContextPtr /* context_ */, const String & table_n
|
||||
throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists.", backQuote(database_name), backQuote(table_name));
|
||||
|
||||
it->second.expiration_iterator = cache_expiration_queue.emplace(cache_expiration_queue.end(), current_time, table_name);
|
||||
|
||||
CurrentMetrics::add(CurrentMetrics::AttachedTable, 1);
|
||||
}
|
||||
|
||||
@ -202,6 +203,7 @@ StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & ta
|
||||
if (it->second.expiration_iterator != cache_expiration_queue.end())
|
||||
cache_expiration_queue.erase(it->second.expiration_iterator);
|
||||
tables_cache.erase(it);
|
||||
|
||||
CurrentMetrics::sub(CurrentMetrics::AttachedTable, 1);
|
||||
}
|
||||
return res;
|
||||
|
||||
@ -260,7 +260,9 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
|
||||
res = it->second;
|
||||
tables.erase(it);
|
||||
res->is_detached = true;
|
||||
CurrentMetrics::sub(getAttachedCounterForStorage(res), 1);
|
||||
|
||||
if (res->isSystemStorage() == false)
|
||||
CurrentMetrics::sub(getAttachedCounterForStorage(res), 1);
|
||||
|
||||
auto table_id = res->getStorageID();
|
||||
if (table_id.hasUUID())
|
||||
@ -301,7 +303,9 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
|
||||
/// It is important to reset is_detached here since in case of RENAME in
|
||||
/// non-Atomic database the is_detached is set to true before RENAME.
|
||||
table->is_detached = false;
|
||||
CurrentMetrics::add(getAttachedCounterForStorage(table), 1);
|
||||
|
||||
if (table->isSystemStorage() == false && table_id.database_name != DatabaseCatalog::SYSTEM_DATABASE)
|
||||
CurrentMetrics::add(getAttachedCounterForStorage(table), 1);
|
||||
}
|
||||
|
||||
void DatabaseWithOwnTablesBase::shutdown()
|
||||
|
||||
@ -113,6 +113,36 @@ struct ByteHammingDistanceImpl
|
||||
}
|
||||
};
|
||||
|
||||
void parseUTF8String(const char * __restrict data, size_t size, std::function<void(UInt32)> utf8_consumer, std::function<void(unsigned char)> ascii_consumer = nullptr)
|
||||
{
|
||||
const char * end = data + size;
|
||||
while (data < end)
|
||||
{
|
||||
size_t len = UTF8::seqLength(*data);
|
||||
if (len == 1)
|
||||
{
|
||||
if (ascii_consumer)
|
||||
ascii_consumer(static_cast<unsigned char>(*data));
|
||||
else
|
||||
utf8_consumer(static_cast<UInt32>(*data));
|
||||
++data;
|
||||
}
|
||||
else
|
||||
{
|
||||
auto code_point = UTF8::convertUTF8ToCodePoint(data, end - data);
|
||||
if (code_point.has_value())
|
||||
{
|
||||
utf8_consumer(code_point.value());
|
||||
data += len;
|
||||
}
|
||||
else
|
||||
{
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Illegal UTF-8 sequence, while processing '{}'", StringRef(data, end - data));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <bool is_utf8>
|
||||
struct ByteJaccardIndexImpl
|
||||
{
|
||||
@ -138,57 +168,28 @@ struct ByteJaccardIndexImpl
|
||||
haystack_set.fill(0);
|
||||
needle_set.fill(0);
|
||||
|
||||
while (haystack < haystack_end)
|
||||
if constexpr (is_utf8)
|
||||
{
|
||||
size_t len = 1;
|
||||
if constexpr (is_utf8)
|
||||
len = UTF8::seqLength(*haystack);
|
||||
|
||||
if (len == 1)
|
||||
parseUTF8String(
|
||||
haystack,
|
||||
haystack_size,
|
||||
[&](UInt32 data) { haystack_utf8_set.insert(data); },
|
||||
[&](unsigned char data) { haystack_set[data] = 1; });
|
||||
parseUTF8String(
|
||||
needle, needle_size, [&](UInt32 data) { needle_utf8_set.insert(data); }, [&](unsigned char data) { needle_set[data] = 1; });
|
||||
}
|
||||
else
|
||||
{
|
||||
while (haystack < haystack_end)
|
||||
{
|
||||
haystack_set[static_cast<unsigned char>(*haystack)] = 1;
|
||||
++haystack;
|
||||
}
|
||||
else
|
||||
{
|
||||
auto code_point = UTF8::convertUTF8ToCodePoint(haystack, haystack_end - haystack);
|
||||
if (code_point.has_value())
|
||||
{
|
||||
haystack_utf8_set.insert(code_point.value());
|
||||
haystack += len;
|
||||
}
|
||||
else
|
||||
{
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Illegal UTF-8 sequence, while processing '{}'", StringRef(haystack, haystack_end - haystack));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
while (needle < needle_end)
|
||||
{
|
||||
|
||||
size_t len = 1;
|
||||
if constexpr (is_utf8)
|
||||
len = UTF8::seqLength(*needle);
|
||||
|
||||
if (len == 1)
|
||||
while (needle < needle_end)
|
||||
{
|
||||
needle_set[static_cast<unsigned char>(*needle)] = 1;
|
||||
++needle;
|
||||
}
|
||||
else
|
||||
{
|
||||
auto code_point = UTF8::convertUTF8ToCodePoint(needle, needle_end - needle);
|
||||
if (code_point.has_value())
|
||||
{
|
||||
needle_utf8_set.insert(code_point.value());
|
||||
needle += len;
|
||||
}
|
||||
else
|
||||
{
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Illegal UTF-8 sequence, while processing '{}'", StringRef(needle, needle_end - needle));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
UInt8 intersection = 0;
|
||||
@ -226,6 +227,7 @@ struct ByteJaccardIndexImpl
|
||||
|
||||
static constexpr size_t max_string_size = 1u << 16;
|
||||
|
||||
template<bool is_utf8>
|
||||
struct ByteEditDistanceImpl
|
||||
{
|
||||
using ResultType = UInt64;
|
||||
@ -242,6 +244,16 @@ struct ByteEditDistanceImpl
|
||||
ErrorCodes::TOO_LARGE_STRING_SIZE,
|
||||
"The string size is too big for function editDistance, should be at most {}", max_string_size);
|
||||
|
||||
PaddedPODArray<UInt32> haystack_utf8;
|
||||
PaddedPODArray<UInt32> needle_utf8;
|
||||
if constexpr (is_utf8)
|
||||
{
|
||||
parseUTF8String(haystack, haystack_size, [&](UInt32 data) { haystack_utf8.push_back(data); });
|
||||
parseUTF8String(needle, needle_size, [&](UInt32 data) { needle_utf8.push_back(data); });
|
||||
haystack_size = haystack_utf8.size();
|
||||
needle_size = needle_utf8.size();
|
||||
}
|
||||
|
||||
PaddedPODArray<ResultType> distances0(haystack_size + 1, 0);
|
||||
PaddedPODArray<ResultType> distances1(haystack_size + 1, 0);
|
||||
|
||||
@ -261,9 +273,16 @@ struct ByteEditDistanceImpl
|
||||
insertion = distances1[pos_haystack] + 1;
|
||||
substitution = distances0[pos_haystack];
|
||||
|
||||
if (*(needle + pos_needle) != *(haystack + pos_haystack))
|
||||
substitution += 1;
|
||||
|
||||
if constexpr (is_utf8)
|
||||
{
|
||||
if (needle_utf8[pos_needle] != haystack_utf8[pos_haystack])
|
||||
substitution += 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (*(needle + pos_needle) != *(haystack + pos_haystack))
|
||||
substitution += 1;
|
||||
}
|
||||
distances1[pos_haystack + 1] = std::min(deletion, std::min(substitution, insertion));
|
||||
}
|
||||
distances0.swap(distances1);
|
||||
@ -457,7 +476,12 @@ struct NameEditDistance
|
||||
{
|
||||
static constexpr auto name = "editDistance";
|
||||
};
|
||||
using FunctionEditDistance = FunctionsStringSimilarity<FunctionStringDistanceImpl<ByteEditDistanceImpl>, NameEditDistance>;
|
||||
using FunctionEditDistance = FunctionsStringSimilarity<FunctionStringDistanceImpl<ByteEditDistanceImpl<false>>, NameEditDistance>;
|
||||
struct NameEditDistanceUTF8
|
||||
{
|
||||
static constexpr auto name = "editDistanceUTF8";
|
||||
};
|
||||
using FunctionEditDistanceUTF8 = FunctionsStringSimilarity<FunctionStringDistanceImpl<ByteEditDistanceImpl<true>>, NameEditDistanceUTF8>;
|
||||
|
||||
struct NameDamerauLevenshteinDistance
|
||||
{
|
||||
@ -499,6 +523,10 @@ REGISTER_FUNCTION(StringDistance)
|
||||
FunctionDocumentation{.description = R"(Calculates the edit distance between two byte-strings.)"});
|
||||
factory.registerAlias("levenshteinDistance", NameEditDistance::name);
|
||||
|
||||
factory.registerFunction<FunctionEditDistanceUTF8>(
|
||||
FunctionDocumentation{.description = R"(Calculates the edit distance between two UTF8 strings.)"});
|
||||
factory.registerAlias("levenshteinDistanceUTF8", NameEditDistanceUTF8::name);
|
||||
|
||||
factory.registerFunction<FunctionDamerauLevenshteinDistance>(
|
||||
FunctionDocumentation{.description = R"(Calculates the Damerau-Levenshtein distance two between two byte-string.)"});
|
||||
|
||||
|
||||
181
src/Functions/dateTimeToSnowflakeID.cpp
Normal file
181
src/Functions/dateTimeToSnowflakeID.cpp
Normal file
@ -0,0 +1,181 @@
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Columns/ColumnsDateTime.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int UNKNOWN_FUNCTION;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/// See generateSnowflakeID.cpp
|
||||
constexpr int time_shift = 22;
|
||||
|
||||
}
|
||||
|
||||
class FunctionDateTimeToSnowflakeID : public IFunction
|
||||
{
|
||||
private:
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
static constexpr auto name = "dateTimeToSnowflakeID";
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionDateTimeToSnowflakeID>(context); }
|
||||
explicit FunctionDateTimeToSnowflakeID(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool isVariadic() const override { return true; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateTime), nullptr, "DateTime"}
|
||||
};
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"epoch", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), isColumnConst, "UInt*"}
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args, optional_args);
|
||||
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (!uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "To use function {}, setting 'uniform_snowflake_conversion_functions' must be enabled", getName());
|
||||
|
||||
const auto & col_src = *arguments[0].column;
|
||||
|
||||
size_t epoch = 0;
|
||||
if (arguments.size() == 2 && input_rows_count != 0)
|
||||
{
|
||||
const auto & col_epoch = *arguments[1].column;
|
||||
epoch = col_epoch.getUInt(0);
|
||||
}
|
||||
|
||||
auto col_res = ColumnUInt64::create(input_rows_count);
|
||||
auto & res_data = col_res->getData();
|
||||
|
||||
const auto & src_data = typeid_cast<const ColumnDateTime &>(col_src).getData();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = (static_cast<UInt64>(src_data[i]) * 1000 - epoch) << time_shift;
|
||||
return col_res;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class FunctionDateTime64ToSnowflakeID : public IFunction
|
||||
{
|
||||
private:
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
static constexpr auto name = "dateTime64ToSnowflakeID";
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionDateTime64ToSnowflakeID>(context); }
|
||||
explicit FunctionDateTime64ToSnowflakeID(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool isVariadic() const override { return true; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateTime64), nullptr, "DateTime64"}
|
||||
};
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"epoch", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), isColumnConst, "UInt*"}
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args, optional_args);
|
||||
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (!uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "To use function {}, setting 'uniform_snowflake_conversion_functions' must be enabled", getName());
|
||||
|
||||
const auto & col_src = *arguments[0].column;
|
||||
const auto & src_data = typeid_cast<const ColumnDateTime64 &>(col_src).getData();
|
||||
|
||||
size_t epoch = 0;
|
||||
if (arguments.size() == 2 && input_rows_count != 0)
|
||||
{
|
||||
const auto & col_epoch = *arguments[1].column;
|
||||
epoch = col_epoch.getUInt(0);
|
||||
}
|
||||
|
||||
auto col_res = ColumnUInt64::create(input_rows_count);
|
||||
auto & res_data = col_res->getData();
|
||||
|
||||
/// timestamps in snowflake-ids are millisecond-based, convert input to milliseconds
|
||||
UInt32 src_scale = getDecimalScale(*arguments[0].type);
|
||||
Int64 multiplier_msec = DecimalUtils::scaleMultiplier<DateTime64>(3);
|
||||
Int64 multiplier_src = DecimalUtils::scaleMultiplier<DateTime64>(src_scale);
|
||||
auto factor = multiplier_msec / static_cast<double>(multiplier_src);
|
||||
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = static_cast<UInt64>(src_data[i] * factor - epoch) << time_shift;
|
||||
|
||||
return col_res;
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_FUNCTION(DateTimeToSnowflakeID)
|
||||
{
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Converts a [DateTime](../data-types/datetime.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.)";
|
||||
FunctionDocumentation::Syntax syntax = "dateTimeToSnowflakeID(value[, epoch])";
|
||||
FunctionDocumentation::Arguments arguments = {
|
||||
{"value", "Date with time. [DateTime](../data-types/datetime.md)."},
|
||||
{"epoch", "Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md)"}
|
||||
};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.";
|
||||
FunctionDocumentation::Examples examples = {{"simple", "SELECT dateTimeToSnowflakeID(toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai'))", "6832626392367104000"}};
|
||||
FunctionDocumentation::Categories categories = {"Snowflake ID"};
|
||||
|
||||
factory.registerFunction<FunctionDateTimeToSnowflakeID>({description, syntax, arguments, returned_value, examples, categories});
|
||||
}
|
||||
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Converts a [DateTime64](../data-types/datetime64.md) value to the first [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) at the giving time.)";
|
||||
FunctionDocumentation::Syntax syntax = "dateTime64ToSnowflakeID(value[, epoch])";
|
||||
FunctionDocumentation::Arguments arguments = {
|
||||
{"value", "Date with time. [DateTime64](../data-types/datetime.md)."},
|
||||
{"epoch", "Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md)"}
|
||||
};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "Input value converted to [UInt64](../data-types/int-uint.md) as the first Snowflake ID at that time.";
|
||||
FunctionDocumentation::Examples examples = {{"simple", "SELECT dateTime64ToSnowflakeID(toDateTime64('2021-08-15 18:57:56', 3, 'Asia/Shanghai'))", "6832626394434895872"}};
|
||||
FunctionDocumentation::Categories categories = {"Snowflake ID"};
|
||||
|
||||
factory.registerFunction<FunctionDateTime64ToSnowflakeID>({description, syntax, arguments, returned_value, examples, categories});
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -207,7 +207,7 @@ public:
|
||||
|
||||
REGISTER_FUNCTION(GenerateSnowflakeID)
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Generates a Snowflake ID. The generated Snowflake ID contains the current Unix timestamp in milliseconds 41 (+ 1 top zero bit) bits, followed by machine id (10 bits), a counter (12 bits) to distinguish IDs within a millisecond. For any given timestamp (unix_ts_ms), the counter starts at 0 and is incremented by 1 for each new Snowflake ID until the timestamp changes. In case the counter overflows, the timestamp field is incremented by 1 and the counter is reset to 0. Function generateSnowflakeID guarantees that the counter field within a timestamp increments monotonically across all function invocations in concurrently running threads and queries.)";
|
||||
FunctionDocumentation::Description description = R"(Generates a Snowflake ID. The generated Snowflake ID contains the current Unix timestamp in milliseconds (41 + 1 top zero bits), followed by a machine id (10 bits), and a counter (12 bits) to distinguish IDs within a millisecond. For any given timestamp (unix_ts_ms), the counter starts at 0 and is incremented by 1 for each new Snowflake ID until the timestamp changes. In case the counter overflows, the timestamp field is incremented by 1 and the counter is reset to 0. Function generateSnowflakeID guarantees that the counter field within a timestamp increments monotonically across all function invocations in concurrently running threads and queries.)";
|
||||
FunctionDocumentation::Syntax syntax = "generateSnowflakeID([expression])";
|
||||
FunctionDocumentation::Arguments arguments = {{"expression", "The expression is used to bypass common subexpression elimination if the function is called multiple times in a query but otherwise ignored. Optional."}};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "A value of type UInt64";
|
||||
|
||||
@ -11,11 +11,17 @@
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
|
||||
/// ------------------------------------------------------------------------------------------------------------------------------
|
||||
/// The functions in this file are deprecated and should be removed in favor of functions 'snowflakeIDToDateTime[64]' and
|
||||
/// 'dateTime[64]ToSnowflakeID' by summer 2025. Please also mark setting `uniform_snowflake_conversion_functions` as obsolete then.
|
||||
/// ------------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int DEPRECATED_FUNCTION;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -34,10 +40,19 @@ constexpr int time_shift = 22;
|
||||
class FunctionDateTimeToSnowflake : public IFunction
|
||||
{
|
||||
private:
|
||||
const char * name;
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
explicit FunctionDateTimeToSnowflake(const char * name_) : name(name_) { }
|
||||
static constexpr auto name = "dateTimeToSnowflake";
|
||||
|
||||
static FunctionPtr create(ContextPtr context)
|
||||
{
|
||||
return std::make_shared<FunctionDateTimeToSnowflake>(context);
|
||||
}
|
||||
|
||||
explicit FunctionDateTimeToSnowflake(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
@ -56,6 +71,9 @@ public:
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated, to enable it disable setting 'uniform_snowflake_conversion_functions'", getName());
|
||||
|
||||
const auto & src = arguments[0];
|
||||
const auto & src_column = *src.column;
|
||||
|
||||
@ -73,13 +91,20 @@ public:
|
||||
class FunctionSnowflakeToDateTime : public IFunction
|
||||
{
|
||||
private:
|
||||
const char * name;
|
||||
const bool allow_nonconst_timezone_arguments;
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
explicit FunctionSnowflakeToDateTime(const char * name_, ContextPtr context)
|
||||
: name(name_)
|
||||
, allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments)
|
||||
static constexpr auto name = "snowflakeToDateTime";
|
||||
|
||||
static FunctionPtr create(ContextPtr context)
|
||||
{
|
||||
return std::make_shared<FunctionSnowflakeToDateTime>(context);
|
||||
}
|
||||
|
||||
explicit FunctionSnowflakeToDateTime(ContextPtr context)
|
||||
: allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments)
|
||||
, uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
@ -107,6 +132,9 @@ public:
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated, to enable it disable setting 'uniform_snowflake_conversion_functions'", getName());
|
||||
|
||||
const auto & src = arguments[0];
|
||||
const auto & src_column = *src.column;
|
||||
|
||||
@ -138,10 +166,19 @@ public:
|
||||
class FunctionDateTime64ToSnowflake : public IFunction
|
||||
{
|
||||
private:
|
||||
const char * name;
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
explicit FunctionDateTime64ToSnowflake(const char * name_) : name(name_) { }
|
||||
static constexpr auto name = "dateTime64ToSnowflake";
|
||||
|
||||
static FunctionPtr create(ContextPtr context)
|
||||
{
|
||||
return std::make_shared<FunctionDateTime64ToSnowflake>(context);
|
||||
}
|
||||
|
||||
explicit FunctionDateTime64ToSnowflake(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
@ -160,6 +197,9 @@ public:
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated, to enable it disable setting 'uniform_snowflake_conversion_functions'", getName());
|
||||
|
||||
const auto & src = arguments[0];
|
||||
|
||||
const auto & src_column = *src.column;
|
||||
@ -185,13 +225,20 @@ public:
|
||||
class FunctionSnowflakeToDateTime64 : public IFunction
|
||||
{
|
||||
private:
|
||||
const char * name;
|
||||
const bool allow_nonconst_timezone_arguments;
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
|
||||
public:
|
||||
explicit FunctionSnowflakeToDateTime64(const char * name_, ContextPtr context)
|
||||
: name(name_)
|
||||
, allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments)
|
||||
static constexpr auto name = "snowflakeToDateTime64";
|
||||
|
||||
static FunctionPtr create(ContextPtr context)
|
||||
{
|
||||
return std::make_shared<FunctionSnowflakeToDateTime64>(context);
|
||||
}
|
||||
|
||||
explicit FunctionSnowflakeToDateTime64(ContextPtr context)
|
||||
: allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments)
|
||||
, uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
@ -219,6 +266,9 @@ public:
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated, to enable it disable setting 'uniform_snowflake_conversion_functions'", getName());
|
||||
|
||||
const auto & src = arguments[0];
|
||||
const auto & src_column = *src.column;
|
||||
|
||||
@ -246,27 +296,12 @@ public:
|
||||
|
||||
}
|
||||
|
||||
REGISTER_FUNCTION(DateTimeToSnowflake)
|
||||
REGISTER_FUNCTION(LegacySnowflakeConversion)
|
||||
{
|
||||
factory.registerFunction("dateTimeToSnowflake",
|
||||
[](ContextPtr){ return std::make_shared<FunctionDateTimeToSnowflake>("dateTimeToSnowflake"); });
|
||||
}
|
||||
|
||||
REGISTER_FUNCTION(DateTime64ToSnowflake)
|
||||
{
|
||||
factory.registerFunction("dateTime64ToSnowflake",
|
||||
[](ContextPtr){ return std::make_shared<FunctionDateTime64ToSnowflake>("dateTime64ToSnowflake"); });
|
||||
}
|
||||
|
||||
REGISTER_FUNCTION(SnowflakeToDateTime)
|
||||
{
|
||||
factory.registerFunction("snowflakeToDateTime",
|
||||
[](ContextPtr context){ return std::make_shared<FunctionSnowflakeToDateTime>("snowflakeToDateTime", context); });
|
||||
}
|
||||
REGISTER_FUNCTION(SnowflakeToDateTime64)
|
||||
{
|
||||
factory.registerFunction("snowflakeToDateTime64",
|
||||
[](ContextPtr context){ return std::make_shared<FunctionSnowflakeToDateTime64>("snowflakeToDateTime64", context); });
|
||||
factory.registerFunction<FunctionSnowflakeToDateTime>();
|
||||
factory.registerFunction<FunctionSnowflakeToDateTime64>();
|
||||
factory.registerFunction<FunctionDateTimeToSnowflake>();
|
||||
factory.registerFunction<FunctionDateTime64ToSnowflake>();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
217
src/Functions/snowflakeIDToDateTime.cpp
Normal file
217
src/Functions/snowflakeIDToDateTime.cpp
Normal file
@ -0,0 +1,217 @@
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/extractTimeZoneFromFunctionArguments.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Columns/ColumnsDateTime.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int UNKNOWN_FUNCTION;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/// See generateSnowflakeID.cpp
|
||||
constexpr int time_shift = 22;
|
||||
|
||||
}
|
||||
|
||||
class FunctionSnowflakeIDToDateTime : public IFunction
|
||||
{
|
||||
private:
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
const bool allow_nonconst_timezone_arguments;
|
||||
|
||||
public:
|
||||
static constexpr auto name = "snowflakeIDToDateTime";
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionSnowflakeIDToDateTime>(context); }
|
||||
explicit FunctionSnowflakeIDToDateTime(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
, allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool isVariadic() const override { return true; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isUInt64), nullptr, "UInt64"}
|
||||
};
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"epoch", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), isColumnConst, "UInt*"},
|
||||
{"time_zone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args, optional_args);
|
||||
|
||||
String timezone;
|
||||
if (arguments.size() == 3)
|
||||
timezone = extractTimeZoneNameFromFunctionArguments(arguments, 2, 0, allow_nonconst_timezone_arguments);
|
||||
|
||||
return std::make_shared<DataTypeDateTime>(timezone);
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (!uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "To use function {}, setting 'uniform_snowflake_conversion_functions' must be enabled", getName());
|
||||
|
||||
const auto & col_src = *arguments[0].column;
|
||||
|
||||
size_t epoch = 0;
|
||||
if (arguments.size() >= 2 && input_rows_count != 0)
|
||||
{
|
||||
const auto & col_epoch = *arguments[1].column;
|
||||
epoch = col_epoch.getUInt(0);
|
||||
}
|
||||
|
||||
auto col_res = ColumnDateTime::create(input_rows_count);
|
||||
auto & res_data = col_res->getData();
|
||||
|
||||
if (const auto * col_src_non_const = typeid_cast<const ColumnUInt64 *>(&col_src))
|
||||
{
|
||||
const auto & src_data = col_src_non_const->getData();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = static_cast<UInt32>(((src_data[i] >> time_shift) + epoch) / 1000);
|
||||
}
|
||||
else if (const auto * col_src_const = typeid_cast<const ColumnConst *>(&col_src))
|
||||
{
|
||||
UInt64 src_val = col_src_const->getValue<UInt64>();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = static_cast<UInt32>(((src_val >> time_shift) + epoch) / 1000);
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal argument for function {}", name);
|
||||
|
||||
return col_res;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class FunctionSnowflakeIDToDateTime64 : public IFunction
|
||||
{
|
||||
private:
|
||||
const bool uniform_snowflake_conversion_functions;
|
||||
const bool allow_nonconst_timezone_arguments;
|
||||
|
||||
public:
|
||||
static constexpr auto name = "snowflakeIDToDateTime64";
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionSnowflakeIDToDateTime64>(context); }
|
||||
explicit FunctionSnowflakeIDToDateTime64(ContextPtr context)
|
||||
: uniform_snowflake_conversion_functions(context->getSettingsRef().uniform_snowflake_conversion_functions)
|
||||
, allow_nonconst_timezone_arguments(context->getSettings().allow_nonconst_timezone_arguments)
|
||||
{}
|
||||
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
bool isVariadic() const override { return true; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
FunctionArgumentDescriptors args{
|
||||
{"value", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isUInt64), nullptr, "UInt64"}
|
||||
};
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"epoch", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), isColumnConst, "UInt*"},
|
||||
{"time_zone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
|
||||
};
|
||||
validateFunctionArgumentTypes(*this, arguments, args, optional_args);
|
||||
|
||||
String timezone;
|
||||
if (arguments.size() == 3)
|
||||
timezone = extractTimeZoneNameFromFunctionArguments(arguments, 2, 0, allow_nonconst_timezone_arguments);
|
||||
|
||||
return std::make_shared<DataTypeDateTime64>(3, timezone);
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
if (!uniform_snowflake_conversion_functions)
|
||||
throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "To use function {}, setting 'uniform_snowflake_conversion_functions' must be enabled", getName());
|
||||
|
||||
const auto & col_src = *arguments[0].column;
|
||||
|
||||
size_t epoch = 0;
|
||||
if (arguments.size() >= 2 && input_rows_count != 0)
|
||||
{
|
||||
const auto & col_epoch = *arguments[1].column;
|
||||
epoch = col_epoch.getUInt(0);
|
||||
}
|
||||
|
||||
auto col_res = ColumnDateTime64::create(input_rows_count, 3);
|
||||
auto & res_data = col_res->getData();
|
||||
|
||||
if (const auto * col_src_non_const = typeid_cast<const ColumnUInt64 *>(&col_src))
|
||||
{
|
||||
const auto & src_data = col_src_non_const->getData();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = (src_data[i] >> time_shift) + epoch;
|
||||
}
|
||||
else if (const auto * col_src_const = typeid_cast<const ColumnConst *>(&col_src))
|
||||
{
|
||||
UInt64 src_val = col_src_const->getValue<UInt64>();
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
res_data[i] = (src_val >> time_shift) + epoch;
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal argument for function {}", name);
|
||||
|
||||
return col_res;
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_FUNCTION(SnowflakeIDToDateTime)
|
||||
{
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime](../data-types/datetime.md).)";
|
||||
FunctionDocumentation::Syntax syntax = "snowflakeIDToDateTime(value[, epoch[, time_zone]])";
|
||||
FunctionDocumentation::Arguments arguments = {
|
||||
{"value", "Snowflake ID. [UInt64](../data-types/int-uint.md)"},
|
||||
{"epoch", "Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md)"},
|
||||
{"time_zone", "[Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md)"}
|
||||
};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "The timestamp component of `value` as a [DateTime](../data-types/datetime.md) value.";
|
||||
FunctionDocumentation::Examples examples = {{"simple", "SELECT snowflakeIDToDateTime(7204436857747984384)", "2024-06-06 10:59:58"}};
|
||||
FunctionDocumentation::Categories categories = {"Snowflake ID"};
|
||||
|
||||
factory.registerFunction<FunctionSnowflakeIDToDateTime>({description, syntax, arguments, returned_value, examples, categories});
|
||||
}
|
||||
|
||||
{
|
||||
FunctionDocumentation::Description description = R"(Returns the timestamp component of a [Snowflake ID](https://en.wikipedia.org/wiki/Snowflake_ID) as a value of type [DateTime64](../data-types/datetime64.md).)";
|
||||
FunctionDocumentation::Syntax syntax = "snowflakeIDToDateTime64(value[, epoch[, time_zone]])";
|
||||
FunctionDocumentation::Arguments arguments = {
|
||||
{"value", "Snowflake ID. [UInt64](../data-types/int-uint.md)"},
|
||||
{"epoch", "Epoch of the Snowflake ID in milliseconds since 1970-01-01. Defaults to 0 (1970-01-01). For the Twitter/X epoch (2015-01-01), provide 1288834974657. Optional. [UInt*](../data-types/int-uint.md)"},
|
||||
{"time_zone", "[Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../data-types/string.md)"}
|
||||
};
|
||||
FunctionDocumentation::ReturnedValue returned_value = "The timestamp component of `value` as a [DateTime64](../data-types/datetime64.md) with scale = 3, i.e. millisecond precision.";
|
||||
FunctionDocumentation::Examples examples = {{"simple", "SELECT snowflakeIDToDateTime64(7204436857747984384)", "2024-06-06 10:59:58"}};
|
||||
FunctionDocumentation::Categories categories = {"Snowflake ID"};
|
||||
|
||||
factory.registerFunction<FunctionSnowflakeIDToDateTime64>({description, syntax, arguments, returned_value, examples, categories});
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -118,7 +118,7 @@ namespace
|
||||
/// Checks if the current user has enough access rights granted with grant option to grant or revoke specified access rights.
|
||||
void checkGrantOption(
|
||||
const AccessControl & access_control,
|
||||
const ContextAccess & current_user_access,
|
||||
const ContextAccessWrapper & current_user_access,
|
||||
const std::vector<UUID> & grantees_from_query,
|
||||
bool & need_check_grantees_are_allowed,
|
||||
const AccessRightsElements & elements_to_grant,
|
||||
@ -200,7 +200,7 @@ namespace
|
||||
/// Checks if the current user has enough roles granted with admin option to grant or revoke specified roles.
|
||||
void checkAdminOption(
|
||||
const AccessControl & access_control,
|
||||
const ContextAccess & current_user_access,
|
||||
const ContextAccessWrapper & current_user_access,
|
||||
const std::vector<UUID> & grantees_from_query,
|
||||
bool & need_check_grantees_are_allowed,
|
||||
const std::vector<UUID> & roles_to_grant,
|
||||
@ -277,7 +277,7 @@ namespace
|
||||
/// This function is less accurate than checkAdminOption() because it cannot use any information about
|
||||
/// granted roles the grantees currently have (due to those grantees are located on multiple nodes,
|
||||
/// we just don't have the full information about them).
|
||||
void checkAdminOptionForExecutingOnCluster(const ContextAccess & current_user_access,
|
||||
void checkAdminOptionForExecutingOnCluster(const ContextAccessWrapper & current_user_access,
|
||||
const std::vector<UUID> roles_to_grant,
|
||||
const RolesOrUsersSet & roles_to_revoke)
|
||||
{
|
||||
@ -376,7 +376,7 @@ namespace
|
||||
/// Calculates all available rights to grant with current user intersection.
|
||||
void calculateCurrentGrantRightsWithIntersection(
|
||||
AccessRights & rights,
|
||||
std::shared_ptr<const ContextAccess> current_user_access,
|
||||
std::shared_ptr<const ContextAccessWrapper> current_user_access,
|
||||
const AccessRightsElements & elements_to_grant)
|
||||
{
|
||||
AccessRightsElements current_user_grantable_elements;
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/FieldVisitorsAccurateComparison.h>
|
||||
#include <Common/checkStackSize.h>
|
||||
#include <Common/assert_cast.h>
|
||||
|
||||
#include <Core/ColumnNumbers.h>
|
||||
#include <Core/ColumnWithTypeAndName.h>
|
||||
@ -102,7 +103,7 @@ static size_t getTypeDepth(const DataTypePtr & type)
|
||||
/// 33.33 in the set is converted to 33.3, but it is not equal to 33.3 in the column, so the result should still be empty.
|
||||
/// We can not include values that don't represent any possible value from the type of filtered column to the set.
|
||||
template<typename Collection>
|
||||
static Block createBlockFromCollection(const Collection & collection, const DataTypes & types, bool transform_null_in)
|
||||
static Block createBlockFromCollection(const Collection & collection, const DataTypes & value_types, const DataTypes & types, bool transform_null_in)
|
||||
{
|
||||
size_t columns_num = types.size();
|
||||
MutableColumns columns(columns_num);
|
||||
@ -113,11 +114,12 @@ static Block createBlockFromCollection(const Collection & collection, const Data
|
||||
}
|
||||
|
||||
Row tuple_values;
|
||||
for (const auto & value : collection)
|
||||
for (size_t collection_index = 0; collection_index < collection.size(); ++collection_index)
|
||||
{
|
||||
const auto& value = collection[collection_index];
|
||||
if (columns_num == 1)
|
||||
{
|
||||
auto field = convertFieldToTypeStrict(value, *types[0]);
|
||||
auto field = convertFieldToTypeStrict(value, *value_types[collection_index], *types[0]);
|
||||
bool need_insert_null = transform_null_in && types[0]->isNullable();
|
||||
if (field && (!field->isNull() || need_insert_null))
|
||||
columns[0]->insert(*field);
|
||||
@ -130,7 +132,6 @@ static Block createBlockFromCollection(const Collection & collection, const Data
|
||||
|
||||
const auto & tuple = value.template get<const Tuple &>();
|
||||
size_t tuple_size = tuple.size();
|
||||
|
||||
if (tuple_size != columns_num)
|
||||
throw Exception(ErrorCodes::INCORRECT_ELEMENT_OF_SET, "Incorrect size of tuple in set: {} instead of {}",
|
||||
tuple_size, columns_num);
|
||||
@ -138,10 +139,13 @@ static Block createBlockFromCollection(const Collection & collection, const Data
|
||||
if (tuple_values.empty())
|
||||
tuple_values.resize(tuple_size);
|
||||
|
||||
const DataTypePtr & value_type = value_types[collection_index];
|
||||
const DataTypes & tuple_value_type = typeid_cast<const DataTypeTuple *>(value_type.get())->getElements();
|
||||
|
||||
size_t i = 0;
|
||||
for (; i < tuple_size; ++i)
|
||||
{
|
||||
auto converted_field = convertFieldToTypeStrict(tuple[i], *types[i]);
|
||||
auto converted_field = convertFieldToTypeStrict(tuple[i], *tuple_value_type[i], *types[i]);
|
||||
if (!converted_field)
|
||||
break;
|
||||
tuple_values[i] = std::move(*converted_field);
|
||||
@ -317,16 +321,25 @@ Block createBlockForSet(
|
||||
if (left_type_depth == right_type_depth)
|
||||
{
|
||||
Array array{right_arg_value};
|
||||
block = createBlockFromCollection(array, set_element_types, tranform_null_in);
|
||||
DataTypes value_types{right_arg_type};
|
||||
block = createBlockFromCollection(array, value_types, set_element_types, tranform_null_in);
|
||||
}
|
||||
/// 1 in (1, 2); (1, 2) in ((1, 2), (3, 4)); etc.
|
||||
else if (left_type_depth + 1 == right_type_depth)
|
||||
{
|
||||
auto type_index = right_arg_type->getTypeId();
|
||||
if (type_index == TypeIndex::Tuple)
|
||||
block = createBlockFromCollection(right_arg_value.get<const Tuple &>(), set_element_types, tranform_null_in);
|
||||
{
|
||||
const DataTypes & value_types = assert_cast<const DataTypeTuple *>(right_arg_type.get())->getElements();
|
||||
block = createBlockFromCollection(right_arg_value.get<const Tuple &>(), value_types, set_element_types, tranform_null_in);
|
||||
}
|
||||
else if (type_index == TypeIndex::Array)
|
||||
block = createBlockFromCollection(right_arg_value.get<const Array &>(), set_element_types, tranform_null_in);
|
||||
{
|
||||
const auto* right_arg_array_type = assert_cast<const DataTypeArray *>(right_arg_type.get());
|
||||
size_t right_arg_array_size = right_arg_value.get<const Array &>().size();
|
||||
DataTypes value_types(right_arg_array_size, right_arg_array_type->getNestedType());
|
||||
block = createBlockFromCollection(right_arg_value.get<const Array &>(), value_types, set_element_types, tranform_null_in);
|
||||
}
|
||||
else
|
||||
throw_unsupported_type(right_arg_type);
|
||||
}
|
||||
|
||||
@ -835,6 +835,7 @@ ContextMutablePtr Context::createGlobal(ContextSharedPart * shared_part)
|
||||
auto res = std::shared_ptr<Context>(new Context);
|
||||
res->shared = shared_part;
|
||||
res->query_access_info = std::make_shared<QueryAccessInfo>();
|
||||
res->query_privileges_info = std::make_shared<QueryPrivilegesInfo>();
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -1427,7 +1428,7 @@ void Context::checkAccess(const AccessFlags & flags, const StorageID & table_id,
|
||||
void Context::checkAccess(const AccessRightsElement & element) const { checkAccessImpl(element); }
|
||||
void Context::checkAccess(const AccessRightsElements & elements) const { checkAccessImpl(elements); }
|
||||
|
||||
std::shared_ptr<const ContextAccess> Context::getAccess() const
|
||||
std::shared_ptr<const ContextAccessWrapper> Context::getAccess() const
|
||||
{
|
||||
/// A helper function to collect parameters for calculating access rights, called with Context::getLocalSharedLock() acquired.
|
||||
auto get_params = [this]()
|
||||
@ -1444,14 +1445,14 @@ std::shared_ptr<const ContextAccess> Context::getAccess() const
|
||||
{
|
||||
SharedLockGuard lock(mutex);
|
||||
if (access && !need_recalculate_access)
|
||||
return access; /// No need to recalculate access rights.
|
||||
return std::make_shared<const ContextAccessWrapper>(access, shared_from_this()); /// No need to recalculate access rights.
|
||||
|
||||
params.emplace(get_params());
|
||||
|
||||
if (access && (access->getParams() == *params))
|
||||
{
|
||||
need_recalculate_access = false;
|
||||
return access; /// No need to recalculate access rights.
|
||||
return std::make_shared<const ContextAccessWrapper>(access, shared_from_this()); /// No need to recalculate access rights.
|
||||
}
|
||||
}
|
||||
|
||||
@ -1471,7 +1472,7 @@ std::shared_ptr<const ContextAccess> Context::getAccess() const
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
return std::make_shared<const ContextAccessWrapper>(res, shared_from_this());
|
||||
}
|
||||
|
||||
RowPolicyFilterPtr Context::getRowPolicyFilter(const String & database, const String & table_name, RowPolicyFilterType filter_type) const
|
||||
@ -1857,6 +1858,15 @@ void Context::addQueryFactoriesInfo(QueryLogFactories factory_type, const String
|
||||
}
|
||||
}
|
||||
|
||||
void Context::addQueryPrivilegesInfo(const String & privilege, bool granted) const
|
||||
{
|
||||
std::lock_guard lock(query_privileges_info->mutex);
|
||||
if (granted)
|
||||
query_privileges_info->used_privileges.emplace(privilege);
|
||||
else
|
||||
query_privileges_info->missing_privileges.emplace(privilege);
|
||||
}
|
||||
|
||||
static bool findIdentifier(const ASTFunction * function)
|
||||
{
|
||||
if (!function || !function->arguments)
|
||||
@ -2538,6 +2548,7 @@ void Context::makeQueryContext()
|
||||
local_read_query_throttler.reset();
|
||||
local_write_query_throttler.reset();
|
||||
backups_query_throttler.reset();
|
||||
query_privileges_info = std::make_shared<QueryPrivilegesInfo>(*query_privileges_info);
|
||||
}
|
||||
|
||||
void Context::makeQueryContextForMerge(const MergeTreeSettings & merge_tree_settings)
|
||||
|
||||
@ -50,6 +50,7 @@ class ASTSelectQuery;
|
||||
|
||||
struct ContextSharedPart;
|
||||
class ContextAccess;
|
||||
class ContextAccessWrapper;
|
||||
struct User;
|
||||
using UserPtr = std::shared_ptr<const User>;
|
||||
struct SettingsProfilesInfo;
|
||||
@ -403,9 +404,31 @@ public:
|
||||
mutable std::mutex mutex;
|
||||
};
|
||||
|
||||
struct QueryPrivilegesInfo
|
||||
{
|
||||
QueryPrivilegesInfo() = default;
|
||||
|
||||
QueryPrivilegesInfo(const QueryPrivilegesInfo & rhs)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(rhs.mutex);
|
||||
used_privileges = rhs.used_privileges;
|
||||
missing_privileges = rhs.missing_privileges;
|
||||
}
|
||||
|
||||
QueryPrivilegesInfo(QueryPrivilegesInfo && rhs) = delete;
|
||||
|
||||
std::unordered_set<std::string> used_privileges TSA_GUARDED_BY(mutex);
|
||||
std::unordered_set<std::string> missing_privileges TSA_GUARDED_BY(mutex);
|
||||
|
||||
mutable std::mutex mutex;
|
||||
};
|
||||
|
||||
using QueryPrivilegesInfoPtr = std::shared_ptr<QueryPrivilegesInfo>;
|
||||
|
||||
protected:
|
||||
/// Needs to be changed while having const context in factories methods
|
||||
mutable QueryFactoriesInfo query_factories_info;
|
||||
QueryPrivilegesInfoPtr query_privileges_info;
|
||||
/// Query metrics for reading data asynchronously with IAsynchronousReader.
|
||||
mutable std::shared_ptr<AsyncReadCounters> async_read_counters;
|
||||
|
||||
@ -612,7 +635,7 @@ public:
|
||||
void checkAccess(const AccessRightsElement & element) const;
|
||||
void checkAccess(const AccessRightsElements & elements) const;
|
||||
|
||||
std::shared_ptr<const ContextAccess> getAccess() const;
|
||||
std::shared_ptr<const ContextAccessWrapper> getAccess() const;
|
||||
|
||||
RowPolicyFilterPtr getRowPolicyFilter(const String & database, const String & table_name, RowPolicyFilterType filter_type) const;
|
||||
|
||||
@ -741,6 +764,10 @@ public:
|
||||
QueryFactoriesInfo getQueryFactoriesInfo() const;
|
||||
void addQueryFactoriesInfo(QueryLogFactories factory_type, const String & created_object) const;
|
||||
|
||||
const QueryPrivilegesInfo & getQueryPrivilegesInfo() const { return *getQueryPrivilegesInfoPtr(); }
|
||||
QueryPrivilegesInfoPtr getQueryPrivilegesInfoPtr() const { return query_privileges_info; }
|
||||
void addQueryPrivilegesInfo(const String & privilege, bool granted) const;
|
||||
|
||||
/// For table functions s3/file/url/hdfs/input we can use structure from
|
||||
/// insertion table depending on select expression.
|
||||
StoragePtr executeTableFunction(const ASTPtr & table_expression, const ASTSelectQuery * select_query_hint = nullptr);
|
||||
|
||||
@ -129,6 +129,7 @@ public:
|
||||
static constexpr const char * SYSTEM_DATABASE = "system";
|
||||
static constexpr const char * INFORMATION_SCHEMA = "information_schema";
|
||||
static constexpr const char * INFORMATION_SCHEMA_UPPERCASE = "INFORMATION_SCHEMA";
|
||||
static constexpr const char * DEFAULT_DATABASE = "default";
|
||||
|
||||
/// Returns true if a passed name is one of the predefined databases' names.
|
||||
static bool isPredefinedDatabase(std::string_view database_name);
|
||||
|
||||
@ -88,6 +88,11 @@
|
||||
#include <Interpreters/ReplaceQueryParameterVisitor.h>
|
||||
#include <Parsers/QueryParameterVisitor.h>
|
||||
|
||||
namespace CurrentMetrics
|
||||
{
|
||||
extern const Metric AttachedTable;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -113,6 +118,8 @@ namespace ErrorCodes
|
||||
extern const int UNKNOWN_STORAGE;
|
||||
extern const int SYNTAX_ERROR;
|
||||
extern const int SUPPORT_IS_DISABLED;
|
||||
extern const int TOO_MANY_TABLES;
|
||||
extern const int TOO_MANY_DATABASES;
|
||||
}
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
@ -138,6 +145,31 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
|
||||
throw Exception(ErrorCodes::DATABASE_ALREADY_EXISTS, "Database {} already exists.", database_name);
|
||||
}
|
||||
|
||||
auto db_num_limit = getContext()->getGlobalContext()->getServerSettings().max_database_num_to_throw;
|
||||
if (db_num_limit > 0)
|
||||
{
|
||||
size_t db_count = DatabaseCatalog::instance().getDatabases().size();
|
||||
std::vector<String> system_databases = {
|
||||
DatabaseCatalog::TEMPORARY_DATABASE,
|
||||
DatabaseCatalog::SYSTEM_DATABASE,
|
||||
DatabaseCatalog::INFORMATION_SCHEMA,
|
||||
DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE,
|
||||
DatabaseCatalog::DEFAULT_DATABASE
|
||||
};
|
||||
|
||||
for (const auto & system_database : system_databases)
|
||||
{
|
||||
if (db_count > 0 && DatabaseCatalog::instance().isDatabaseExist(system_database))
|
||||
db_count--;
|
||||
}
|
||||
|
||||
if (db_count >= db_num_limit)
|
||||
throw Exception(ErrorCodes::TOO_MANY_DATABASES,
|
||||
"Too many databases in the Clickhouse. "
|
||||
"The limit (setting 'max_database_num_to_throw') is set to {}, current number of databases is {}",
|
||||
db_num_limit, db_count);
|
||||
}
|
||||
|
||||
/// Will write file with database metadata, if needed.
|
||||
String database_name_escaped = escapeForFileName(database_name);
|
||||
fs::path metadata_path = fs::weakly_canonical(getContext()->getPath());
|
||||
@ -1543,6 +1575,17 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
|
||||
}
|
||||
}
|
||||
|
||||
UInt64 table_num_limit = getContext()->getGlobalContext()->getServerSettings().max_table_num_to_throw;
|
||||
if (table_num_limit > 0 && create.getDatabase() != DatabaseCatalog::SYSTEM_DATABASE)
|
||||
{
|
||||
UInt64 table_count = CurrentMetrics::get(CurrentMetrics::AttachedTable);
|
||||
if (table_count >= table_num_limit)
|
||||
throw Exception(ErrorCodes::TOO_MANY_TABLES,
|
||||
"Too many tables in the Clickhouse. "
|
||||
"The limit (setting 'max_table_num_to_throw') is set to {}, current number of tables is {}",
|
||||
table_num_limit, table_count);
|
||||
}
|
||||
|
||||
database->createTable(getContext(), create.getTable(), res, query_ptr);
|
||||
|
||||
/// Move table data to the proper place. Wo do not move data earlier to avoid situations
|
||||
|
||||
@ -26,7 +26,8 @@
|
||||
#include <Processors/Transforms/CountingTransform.h>
|
||||
#include <Processors/Transforms/ExpressionTransform.h>
|
||||
#include <Processors/Transforms/MaterializingTransform.h>
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
#include <Processors/Transforms/PlanSquashingTransform.h>
|
||||
#include <Processors/Transforms/getSourceFromASTInsertQuery.h>
|
||||
#include <Processors/QueryPlan/QueryPlan.h>
|
||||
#include <QueryPipeline/QueryPipelineBuilder.h>
|
||||
@ -625,9 +626,15 @@ BlockIO InterpreterInsertQuery::execute()
|
||||
{
|
||||
bool table_prefers_large_blocks = table->prefersLargeBlocks();
|
||||
|
||||
pipeline.addTransform(std::make_shared<PlanSquashingTransform>(
|
||||
header,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL,
|
||||
presink_chains.size()));
|
||||
|
||||
pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
|
||||
{
|
||||
return std::make_shared<SimpleSquashingChunksTransform>(
|
||||
return std::make_shared<ApplySquashingTransform>(
|
||||
in_header,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL);
|
||||
@ -683,12 +690,20 @@ BlockIO InterpreterInsertQuery::execute()
|
||||
{
|
||||
bool table_prefers_large_blocks = table->prefersLargeBlocks();
|
||||
|
||||
auto squashing = std::make_shared<SimpleSquashingChunksTransform>(
|
||||
chain.getInputHeader(),
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL);
|
||||
auto squashing = std::make_shared<ApplySquashingTransform>(
|
||||
chain.getInputHeader(),
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL);
|
||||
|
||||
chain.addSource(std::move(squashing));
|
||||
|
||||
auto balancing = std::make_shared<PlanSquashingTransform>(
|
||||
chain.getInputHeader(),
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL,
|
||||
presink_chains.size());
|
||||
|
||||
chain.addSource(std::move(balancing));
|
||||
}
|
||||
|
||||
auto context_ptr = getContext();
|
||||
|
||||
@ -1474,6 +1474,9 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
|
||||
if (expressions.hasHaving() && query.group_by_with_totals && (query.group_by_with_rollup || query.group_by_with_cube))
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "WITH TOTALS and WITH ROLLUP or CUBE are not supported together in presence of HAVING");
|
||||
|
||||
if (query.qualify())
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "QUALIFY clause is not supported in the old analyzer");
|
||||
|
||||
if (options.only_analyze)
|
||||
{
|
||||
auto read_nothing = std::make_unique<ReadNothingStep>(source_header);
|
||||
|
||||
@ -136,6 +136,9 @@ ColumnsDescription QueryLogElement::getColumnsDescription()
|
||||
|
||||
{"used_row_policies", array_low_cardinality_string, "The list of row policies names that were used during query execution."},
|
||||
|
||||
{"used_privileges", array_low_cardinality_string, "Privileges which were successfully checked during query execution."},
|
||||
{"missing_privileges", array_low_cardinality_string, "Privileges that are missing during query execution."},
|
||||
|
||||
{"transaction_id", getTransactionIDDataType(), "The identifier of the transaction in scope of which this query was executed."},
|
||||
|
||||
{"query_cache_usage", std::move(query_cache_usage_datatype), "Usage of the query cache during query execution. Values: 'Unknown' = Status unknown, 'None' = The query result was neither written into nor read from the query cache, 'Write' = The query result was written into the query cache, 'Read' = The query result was read from the query cache."},
|
||||
@ -267,6 +270,8 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const
|
||||
auto & column_storage_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
|
||||
auto & column_table_function_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
|
||||
auto & column_row_policies_names = typeid_cast<ColumnArray &>(*columns[i++]);
|
||||
auto & column_used_privileges = typeid_cast<ColumnArray &>(*columns[i++]);
|
||||
auto & column_missing_privileges = typeid_cast<ColumnArray &>(*columns[i++]);
|
||||
|
||||
auto fill_column = [](const auto & data, ColumnArray & column)
|
||||
{
|
||||
@ -290,6 +295,8 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const
|
||||
fill_column(used_storages, column_storage_factory_objects);
|
||||
fill_column(used_table_functions, column_table_function_factory_objects);
|
||||
fill_column(used_row_policies, column_row_policies_names);
|
||||
fill_column(used_privileges, column_used_privileges);
|
||||
fill_column(missing_privileges, column_missing_privileges);
|
||||
}
|
||||
|
||||
columns[i++]->insert(Tuple{tid.start_csn, tid.local_tid, tid.host_id});
|
||||
|
||||
@ -81,6 +81,8 @@ struct QueryLogElement
|
||||
std::unordered_set<String> used_storages;
|
||||
std::unordered_set<String> used_table_functions;
|
||||
std::set<String> used_row_policies;
|
||||
std::unordered_set<String> used_privileges;
|
||||
std::unordered_set<String> missing_privileges;
|
||||
|
||||
Int32 exception_code{}; // because ErrorCodes are int
|
||||
String exception;
|
||||
|
||||
@ -532,7 +532,7 @@ ContextMutablePtr Session::makeSessionContext()
|
||||
session_context->checkSettingsConstraints(settings_from_auth_server, SettingSource::QUERY);
|
||||
session_context->applySettingsChanges(settings_from_auth_server);
|
||||
|
||||
recordLoginSucess(session_context);
|
||||
recordLoginSuccess(session_context);
|
||||
|
||||
return session_context;
|
||||
}
|
||||
@ -596,7 +596,7 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std:
|
||||
{ session_name_ },
|
||||
max_sessions_for_user);
|
||||
|
||||
recordLoginSucess(session_context);
|
||||
recordLoginSuccess(session_context);
|
||||
|
||||
return session_context;
|
||||
}
|
||||
@ -672,13 +672,13 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t
|
||||
user = query_context->getUser();
|
||||
|
||||
/// Interserver does not create session context
|
||||
recordLoginSucess(query_context);
|
||||
recordLoginSuccess(query_context);
|
||||
|
||||
return query_context;
|
||||
}
|
||||
|
||||
|
||||
void Session::recordLoginSucess(ContextPtr login_context) const
|
||||
void Session::recordLoginSuccess(ContextPtr login_context) const
|
||||
{
|
||||
if (notified_session_log_about_login)
|
||||
return;
|
||||
@ -694,7 +694,7 @@ void Session::recordLoginSucess(ContextPtr login_context) const
|
||||
session_log->addLoginSuccess(auth_id,
|
||||
named_session ? named_session->key.second : "",
|
||||
settings,
|
||||
access,
|
||||
access->getAccess(),
|
||||
getClientInfo(),
|
||||
user);
|
||||
}
|
||||
|
||||
@ -102,8 +102,7 @@ public:
|
||||
private:
|
||||
std::shared_ptr<SessionLog> getSessionLog() const;
|
||||
ContextMutablePtr makeQueryContextImpl(const ClientInfo * client_info_to_copy, ClientInfo * client_info_to_move) const;
|
||||
void recordLoginSucess(ContextPtr login_context) const;
|
||||
|
||||
void recordLoginSuccess(ContextPtr login_context) const;
|
||||
|
||||
mutable bool notified_session_log_about_login = false;
|
||||
const UUID auth_id;
|
||||
|
||||
@ -214,7 +214,7 @@ void SessionLog::addLoginSuccess(const UUID & auth_id,
|
||||
const ClientInfo & client_info,
|
||||
const UserPtr & login_user)
|
||||
{
|
||||
DB::SessionLogElement log_entry(auth_id, SESSION_LOGIN_SUCCESS);
|
||||
SessionLogElement log_entry(auth_id, SESSION_LOGIN_SUCCESS);
|
||||
log_entry.client_info = client_info;
|
||||
|
||||
if (login_user)
|
||||
|
||||
159
src/Interpreters/Squashing.cpp
Normal file
159
src/Interpreters/Squashing.cpp
Normal file
@ -0,0 +1,159 @@
|
||||
#include <vector>
|
||||
#include <Interpreters/Squashing.h>
|
||||
#include <Common/CurrentThread.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
Squashing::Squashing(Block header_, size_t min_block_size_rows_, size_t min_block_size_bytes_)
|
||||
: header(header_)
|
||||
, min_block_size_rows(min_block_size_rows_)
|
||||
, min_block_size_bytes(min_block_size_bytes_)
|
||||
{
|
||||
}
|
||||
|
||||
Chunk Squashing::flush()
|
||||
{
|
||||
return convertToChunk(std::move(chunks_to_merge_vec));
|
||||
}
|
||||
|
||||
Chunk Squashing::squash(Chunk && input_chunk)
|
||||
{
|
||||
if (!input_chunk.hasChunkInfo())
|
||||
return Chunk();
|
||||
|
||||
const auto *info = getInfoFromChunk(input_chunk);
|
||||
return squash(info->chunks);
|
||||
}
|
||||
|
||||
Chunk Squashing::add(Chunk && input_chunk)
|
||||
{
|
||||
if (!input_chunk)
|
||||
return {};
|
||||
|
||||
/// Just read block is already enough.
|
||||
if (isEnoughSize(input_chunk.getNumRows(), input_chunk.bytes()))
|
||||
{
|
||||
/// If no accumulated data, return just read block.
|
||||
if (chunks_to_merge_vec.empty())
|
||||
{
|
||||
chunks_to_merge_vec.push_back(std::move(input_chunk));
|
||||
Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
|
||||
chunks_to_merge_vec.clear();
|
||||
return res_chunk;
|
||||
}
|
||||
|
||||
/// Return accumulated data (maybe it has small size) and place new block to accumulated data.
|
||||
Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
|
||||
chunks_to_merge_vec.clear();
|
||||
changeCurrentSize(input_chunk.getNumRows(), input_chunk.bytes());
|
||||
chunks_to_merge_vec.push_back(std::move(input_chunk));
|
||||
return res_chunk;
|
||||
}
|
||||
|
||||
/// Accumulated block is already enough.
|
||||
if (isEnoughSize(accumulated_size.rows, accumulated_size.bytes))
|
||||
{
|
||||
/// Return accumulated data and place new block to accumulated data.
|
||||
Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
|
||||
chunks_to_merge_vec.clear();
|
||||
changeCurrentSize(input_chunk.getNumRows(), input_chunk.bytes());
|
||||
chunks_to_merge_vec.push_back(std::move(input_chunk));
|
||||
return res_chunk;
|
||||
}
|
||||
|
||||
/// Pushing data into accumulating vector
|
||||
expandCurrentSize(input_chunk.getNumRows(), input_chunk.bytes());
|
||||
chunks_to_merge_vec.push_back(std::move(input_chunk));
|
||||
|
||||
/// If accumulated data is big enough, we send it
|
||||
if (isEnoughSize(accumulated_size.rows, accumulated_size.bytes))
|
||||
{
|
||||
Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
|
||||
changeCurrentSize(0, 0);
|
||||
chunks_to_merge_vec.clear();
|
||||
return res_chunk;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
Chunk Squashing::convertToChunk(std::vector<Chunk> && chunks) const
|
||||
{
|
||||
if (chunks.empty())
|
||||
return {};
|
||||
|
||||
auto info = std::make_shared<ChunksToSquash>();
|
||||
info->chunks = std::move(chunks);
|
||||
|
||||
chunks.clear();
|
||||
|
||||
return Chunk(header.cloneEmptyColumns(), 0, info);
|
||||
}
|
||||
|
||||
Chunk Squashing::squash(std::vector<Chunk> & input_chunks)
|
||||
{
|
||||
Chunk accumulated_chunk;
|
||||
std::vector<IColumn::MutablePtr> mutable_columns = {};
|
||||
size_t rows = 0;
|
||||
for (const Chunk & chunk : input_chunks)
|
||||
rows += chunk.getNumRows();
|
||||
|
||||
{
|
||||
auto & first_chunk = input_chunks[0];
|
||||
Columns columns = first_chunk.detachColumns();
|
||||
for (auto & column : columns)
|
||||
{
|
||||
mutable_columns.push_back(IColumn::mutate(std::move(column)));
|
||||
mutable_columns.back()->reserve(rows);
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 1; i < input_chunks.size(); ++i) // We've already processed the first chunk above
|
||||
{
|
||||
Columns columns = input_chunks[i].detachColumns();
|
||||
for (size_t j = 0, size = mutable_columns.size(); j < size; ++j)
|
||||
{
|
||||
const auto source_column = columns[j];
|
||||
|
||||
mutable_columns[j]->insertRangeFrom(*source_column, 0, source_column->size());
|
||||
}
|
||||
}
|
||||
accumulated_chunk.setColumns(std::move(mutable_columns), rows);
|
||||
return accumulated_chunk;
|
||||
}
|
||||
|
||||
const ChunksToSquash* Squashing::getInfoFromChunk(const Chunk & chunk)
|
||||
{
|
||||
const auto& info = chunk.getChunkInfo();
|
||||
const auto * agg_info = typeid_cast<const ChunksToSquash *>(info.get());
|
||||
|
||||
if (!agg_info)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no ChunksToSquash in ChunkInfoPtr");
|
||||
|
||||
return agg_info;
|
||||
}
|
||||
|
||||
void Squashing::expandCurrentSize(size_t rows, size_t bytes)
|
||||
{
|
||||
accumulated_size.rows += rows;
|
||||
accumulated_size.bytes += bytes;
|
||||
}
|
||||
|
||||
void Squashing::changeCurrentSize(size_t rows, size_t bytes)
|
||||
{
|
||||
accumulated_size.rows = rows;
|
||||
accumulated_size.bytes = bytes;
|
||||
}
|
||||
|
||||
bool Squashing::isEnoughSize(size_t rows, size_t bytes) const
|
||||
{
|
||||
return (!min_block_size_rows && !min_block_size_bytes)
|
||||
|| (min_block_size_rows && rows >= min_block_size_rows)
|
||||
|| (min_block_size_bytes && bytes >= min_block_size_bytes);
|
||||
}
|
||||
}
|
||||
69
src/Interpreters/Squashing.h
Normal file
69
src/Interpreters/Squashing.h
Normal file
@ -0,0 +1,69 @@
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <Core/Block.h>
|
||||
#include <Processors/Chunk.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
struct ChunksToSquash : public ChunkInfo
|
||||
{
|
||||
mutable std::vector<Chunk> chunks = {};
|
||||
};
|
||||
|
||||
/** Merging consecutive passed blocks to specified minimum size.
|
||||
*
|
||||
* (But if one of input blocks has already at least specified size,
|
||||
* then don't merge it with neighbours, even if neighbours are small.)
|
||||
*
|
||||
* Used to prepare blocks to adequate size for INSERT queries,
|
||||
* because such storages as Memory, StripeLog, Log, TinyLog...
|
||||
* store or compress data in blocks exactly as passed to it,
|
||||
* and blocks of small size are not efficient.
|
||||
*
|
||||
* Order of data is kept.
|
||||
*/
|
||||
|
||||
class Squashing
|
||||
{
|
||||
public:
|
||||
explicit Squashing(Block header_, size_t min_block_size_rows_, size_t min_block_size_bytes_);
|
||||
Squashing(Squashing && other) = default;
|
||||
|
||||
Chunk add(Chunk && input_chunk);
|
||||
static Chunk squash(Chunk && input_chunk);
|
||||
Chunk flush();
|
||||
|
||||
bool isDataLeft()
|
||||
{
|
||||
return !chunks_to_merge_vec.empty();
|
||||
}
|
||||
|
||||
Block header;
|
||||
private:
|
||||
struct CurrentSize
|
||||
{
|
||||
size_t rows = 0;
|
||||
size_t bytes = 0;
|
||||
};
|
||||
|
||||
std::vector<Chunk> chunks_to_merge_vec = {};
|
||||
size_t min_block_size_rows;
|
||||
size_t min_block_size_bytes;
|
||||
|
||||
CurrentSize accumulated_size;
|
||||
|
||||
static const ChunksToSquash * getInfoFromChunk(const Chunk & chunk);
|
||||
|
||||
static Chunk squash(std::vector<Chunk> & input_chunks);
|
||||
|
||||
void expandCurrentSize(size_t rows, size_t bytes);
|
||||
void changeCurrentSize(size_t rows, size_t bytes);
|
||||
bool isEnoughSize(size_t rows, size_t bytes) const;
|
||||
|
||||
Chunk convertToChunk(std::vector<Chunk> && chunks) const;
|
||||
};
|
||||
|
||||
}
|
||||
@ -1,145 +0,0 @@
|
||||
#include <Interpreters/SquashingTransform.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
SquashingTransform::SquashingTransform(size_t min_block_size_rows_, size_t min_block_size_bytes_)
|
||||
: min_block_size_rows(min_block_size_rows_)
|
||||
, min_block_size_bytes(min_block_size_bytes_)
|
||||
{
|
||||
}
|
||||
|
||||
Block SquashingTransform::add(Block && input_block)
|
||||
{
|
||||
return addImpl<Block &&>(std::move(input_block));
|
||||
}
|
||||
|
||||
Block SquashingTransform::add(const Block & input_block)
|
||||
{
|
||||
return addImpl<const Block &>(input_block);
|
||||
}
|
||||
|
||||
/*
|
||||
* To minimize copying, accept two types of argument: const reference for output
|
||||
* stream, and rvalue reference for input stream, and decide whether to copy
|
||||
* inside this function. This allows us not to copy Block unless we absolutely
|
||||
* have to.
|
||||
*/
|
||||
template <typename ReferenceType>
|
||||
Block SquashingTransform::addImpl(ReferenceType input_block)
|
||||
{
|
||||
/// End of input stream.
|
||||
if (!input_block)
|
||||
{
|
||||
Block to_return;
|
||||
std::swap(to_return, accumulated_block);
|
||||
return to_return;
|
||||
}
|
||||
|
||||
/// Just read block is already enough.
|
||||
if (isEnoughSize(input_block))
|
||||
{
|
||||
/// If no accumulated data, return just read block.
|
||||
if (!accumulated_block)
|
||||
{
|
||||
return std::move(input_block);
|
||||
}
|
||||
|
||||
/// Return accumulated data (maybe it has small size) and place new block to accumulated data.
|
||||
Block to_return = std::move(input_block);
|
||||
std::swap(to_return, accumulated_block);
|
||||
return to_return;
|
||||
}
|
||||
|
||||
/// Accumulated block is already enough.
|
||||
if (isEnoughSize(accumulated_block))
|
||||
{
|
||||
/// Return accumulated data and place new block to accumulated data.
|
||||
Block to_return = std::move(input_block);
|
||||
std::swap(to_return, accumulated_block);
|
||||
return to_return;
|
||||
}
|
||||
|
||||
append<ReferenceType>(std::move(input_block));
|
||||
if (isEnoughSize(accumulated_block))
|
||||
{
|
||||
Block to_return;
|
||||
std::swap(to_return, accumulated_block);
|
||||
return to_return;
|
||||
}
|
||||
|
||||
/// Squashed block is not ready.
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
template <typename ReferenceType>
|
||||
void SquashingTransform::append(ReferenceType input_block)
|
||||
{
|
||||
if (!accumulated_block)
|
||||
{
|
||||
accumulated_block = std::move(input_block);
|
||||
return;
|
||||
}
|
||||
|
||||
assert(blocksHaveEqualStructure(input_block, accumulated_block));
|
||||
|
||||
try
|
||||
{
|
||||
for (size_t i = 0, size = accumulated_block.columns(); i < size; ++i)
|
||||
{
|
||||
const auto source_column = input_block.getByPosition(i).column;
|
||||
|
||||
auto mutable_column = IColumn::mutate(std::move(accumulated_block.getByPosition(i).column));
|
||||
mutable_column->insertRangeFrom(*source_column, 0, source_column->size());
|
||||
accumulated_block.getByPosition(i).column = std::move(mutable_column);
|
||||
}
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
/// add() may be called again even after a previous add() threw an exception.
|
||||
/// Keep accumulated_block in a valid state.
|
||||
/// Seems ok to discard accumulated data because we're throwing an exception, which the caller will
|
||||
/// hopefully interpret to mean "this block and all *previous* blocks are potentially lost".
|
||||
accumulated_block.clear();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
bool SquashingTransform::isEnoughSize(const Block & block)
|
||||
{
|
||||
size_t rows = 0;
|
||||
size_t bytes = 0;
|
||||
|
||||
for (const auto & [column, type, name] : block)
|
||||
{
|
||||
if (!column)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid column in block.");
|
||||
|
||||
if (!rows)
|
||||
rows = column->size();
|
||||
else if (rows != column->size())
|
||||
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Sizes of columns doesn't match");
|
||||
|
||||
bytes += column->byteSize();
|
||||
}
|
||||
|
||||
return isEnoughSize(rows, bytes);
|
||||
}
|
||||
|
||||
|
||||
bool SquashingTransform::isEnoughSize(size_t rows, size_t bytes) const
|
||||
{
|
||||
return (!min_block_size_rows && !min_block_size_bytes)
|
||||
|| (min_block_size_rows && rows >= min_block_size_rows)
|
||||
|| (min_block_size_bytes && bytes >= min_block_size_bytes);
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,50 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Block.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
/** Merging consecutive passed blocks to specified minimum size.
|
||||
*
|
||||
* (But if one of input blocks has already at least specified size,
|
||||
* then don't merge it with neighbours, even if neighbours are small.)
|
||||
*
|
||||
* Used to prepare blocks to adequate size for INSERT queries,
|
||||
* because such storages as Memory, StripeLog, Log, TinyLog...
|
||||
* store or compress data in blocks exactly as passed to it,
|
||||
* and blocks of small size are not efficient.
|
||||
*
|
||||
* Order of data is kept.
|
||||
*/
|
||||
class SquashingTransform
|
||||
{
|
||||
public:
|
||||
/// Conditions on rows and bytes are OR-ed. If one of them is zero, then corresponding condition is ignored.
|
||||
SquashingTransform(size_t min_block_size_rows_, size_t min_block_size_bytes_);
|
||||
|
||||
/** Add next block and possibly returns squashed block.
|
||||
* At end, you need to pass empty block. As the result for last (empty) block, you will get last Result with ready = true.
|
||||
*/
|
||||
Block add(Block && block);
|
||||
Block add(const Block & block);
|
||||
|
||||
private:
|
||||
size_t min_block_size_rows;
|
||||
size_t min_block_size_bytes;
|
||||
|
||||
Block accumulated_block;
|
||||
|
||||
template <typename ReferenceType>
|
||||
Block addImpl(ReferenceType block);
|
||||
|
||||
template <typename ReferenceType>
|
||||
void append(ReferenceType block);
|
||||
|
||||
bool isEnoughSize(const Block & block);
|
||||
bool isEnoughSize(size_t rows, size_t bytes) const;
|
||||
};
|
||||
|
||||
}
|
||||
@ -615,9 +615,9 @@ static bool decimalEqualsFloat(Field field, Float64 float_value)
|
||||
return decimal_to_float == float_value;
|
||||
}
|
||||
|
||||
std::optional<Field> convertFieldToTypeStrict(const Field & from_value, const IDataType & to_type)
|
||||
std::optional<Field> convertFieldToTypeStrict(const Field & from_value, const IDataType & from_type, const IDataType & to_type)
|
||||
{
|
||||
Field result_value = convertFieldToType(from_value, to_type);
|
||||
Field result_value = convertFieldToType(from_value, to_type, &from_type);
|
||||
|
||||
if (Field::isDecimal(from_value.getType()) && Field::isDecimal(result_value.getType()))
|
||||
{
|
||||
|
||||
@ -22,6 +22,6 @@ Field convertFieldToTypeOrThrow(const Field & from_value, const IDataType & to_t
|
||||
|
||||
/// Applies stricter rules than convertFieldToType, doesn't allow loss of precision converting to Decimal.
|
||||
/// Returns `Field` if the conversion was successful and the result is equal to the original value, otherwise returns nullopt.
|
||||
std::optional<Field> convertFieldToTypeStrict(const Field & from_value, const IDataType & to_type);
|
||||
std::optional<Field> convertFieldToTypeStrict(const Field & from_value, const IDataType & from_type, const IDataType & to_type);
|
||||
|
||||
}
|
||||
|
||||
@ -44,6 +44,7 @@
|
||||
#include <Formats/FormatFactory.h>
|
||||
#include <Storages/StorageInput.h>
|
||||
|
||||
#include <Access/ContextAccess.h>
|
||||
#include <Access/EnabledQuota.h>
|
||||
#include <Interpreters/ApplyWithGlobalVisitor.h>
|
||||
#include <Interpreters/Context.h>
|
||||
@ -221,6 +222,17 @@ static void logException(ContextPtr context, QueryLogElement & elem, bool log_er
|
||||
LOG_INFO(getLogger("executeQuery"), message);
|
||||
}
|
||||
|
||||
static void
|
||||
addPrivilegesInfoToQueryLogElement(QueryLogElement & element, const ContextPtr context_ptr)
|
||||
{
|
||||
const auto & privileges_info = context_ptr->getQueryPrivilegesInfo();
|
||||
{
|
||||
std::lock_guard lock(privileges_info.mutex);
|
||||
element.used_privileges = privileges_info.used_privileges;
|
||||
element.missing_privileges = privileges_info.missing_privileges;
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
addStatusInfoToQueryLogElement(QueryLogElement & element, const QueryStatusInfo & info, const ASTPtr query_ast, const ContextPtr context_ptr)
|
||||
{
|
||||
@ -286,6 +298,7 @@ addStatusInfoToQueryLogElement(QueryLogElement & element, const QueryStatusInfo
|
||||
}
|
||||
|
||||
element.async_read_counters = context_ptr->getAsyncReadCounters();
|
||||
addPrivilegesInfoToQueryLogElement(element, context_ptr);
|
||||
}
|
||||
|
||||
|
||||
@ -601,6 +614,8 @@ void logExceptionBeforeStart(
|
||||
elem.formatted_query = queryToString(ast);
|
||||
}
|
||||
|
||||
addPrivilegesInfoToQueryLogElement(elem, context);
|
||||
|
||||
// We don't calculate databases, tables and columns when the query isn't able to start
|
||||
|
||||
elem.exception_code = getCurrentExceptionCode();
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
#include <Processors/Sinks/SinkToStorage.h>
|
||||
#include <Processors/Transforms/ExpressionTransform.h>
|
||||
#include <Processors/Transforms/MaterializingTransform.h>
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
#include <Processors/Executors/PullingAsyncPipelineExecutor.h>
|
||||
|
||||
#include <QueryPipeline/QueryPipelineBuilder.h>
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/formatReadable.h>
|
||||
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
|
||||
|
||||
namespace ProfileEvents
|
||||
@ -783,7 +783,7 @@ void AggregatingTransform::initGenerate()
|
||||
{
|
||||
/// Just a reasonable constant, matches default value for the setting `preferred_block_size_bytes`
|
||||
static constexpr size_t oneMB = 1024 * 1024;
|
||||
return std::make_shared<SimpleSquashingChunksTransform>(header, params->params.max_block_size, oneMB);
|
||||
return std::make_shared<SimpleSquashingTransform>(header, params->params.max_block_size, oneMB);
|
||||
});
|
||||
}
|
||||
/// AggregatingTransform::expandPipeline expects single output port.
|
||||
|
||||
63
src/Processors/Transforms/ApplySquashingTransform.h
Normal file
63
src/Processors/Transforms/ApplySquashingTransform.h
Normal file
@ -0,0 +1,63 @@
|
||||
#pragma once
|
||||
#include <Interpreters/Squashing.h>
|
||||
#include <Processors/ISimpleTransform.h>
|
||||
#include <Processors/Sinks/SinkToStorage.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class ApplySquashingTransform : public ExceptionKeepingTransform
|
||||
{
|
||||
public:
|
||||
explicit ApplySquashingTransform(const Block & header, const size_t min_block_size_rows, const size_t min_block_size_bytes)
|
||||
: ExceptionKeepingTransform(header, header, false)
|
||||
, squashing(header, min_block_size_rows, min_block_size_bytes)
|
||||
{
|
||||
}
|
||||
|
||||
String getName() const override { return "ApplySquashingTransform"; }
|
||||
|
||||
void work() override
|
||||
{
|
||||
if (stage == Stage::Exception)
|
||||
{
|
||||
data.chunk.clear();
|
||||
ready_input = false;
|
||||
return;
|
||||
}
|
||||
|
||||
ExceptionKeepingTransform::work();
|
||||
if (finish_chunk)
|
||||
{
|
||||
data.chunk = std::move(finish_chunk);
|
||||
ready_output = true;
|
||||
}
|
||||
}
|
||||
|
||||
protected:
|
||||
void onConsume(Chunk chunk) override
|
||||
{
|
||||
if (auto res_chunk = DB::Squashing::squash(std::move(chunk)))
|
||||
cur_chunk.setColumns(res_chunk.getColumns(), res_chunk.getNumRows());
|
||||
}
|
||||
|
||||
GenerateResult onGenerate() override
|
||||
{
|
||||
GenerateResult res;
|
||||
res.chunk = std::move(cur_chunk);
|
||||
res.is_done = true;
|
||||
return res;
|
||||
}
|
||||
void onFinish() override
|
||||
{
|
||||
auto chunk = DB::Squashing::squash({});
|
||||
finish_chunk.setColumns(chunk.getColumns(), chunk.getNumRows());
|
||||
}
|
||||
|
||||
private:
|
||||
Squashing squashing;
|
||||
Chunk cur_chunk;
|
||||
Chunk finish_chunk;
|
||||
};
|
||||
|
||||
}
|
||||
145
src/Processors/Transforms/PlanSquashingTransform.cpp
Normal file
145
src/Processors/Transforms/PlanSquashingTransform.cpp
Normal file
@ -0,0 +1,145 @@
|
||||
#include <Processors/Transforms/PlanSquashingTransform.h>
|
||||
#include <Processors/IProcessor.h>
|
||||
#include <Common/Exception.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
PlanSquashingTransform::PlanSquashingTransform(const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes, size_t num_ports)
|
||||
: IProcessor(InputPorts(num_ports, header), OutputPorts(num_ports, header)), squashing(header, min_block_size_rows, min_block_size_bytes)
|
||||
{
|
||||
}
|
||||
|
||||
IProcessor::Status PlanSquashingTransform::prepare()
|
||||
{
|
||||
Status status = Status::Ready;
|
||||
|
||||
while (planning_status != PlanningStatus::FINISH)
|
||||
{
|
||||
switch (planning_status)
|
||||
{
|
||||
case INIT:
|
||||
init();
|
||||
break;
|
||||
case READ_IF_CAN:
|
||||
return prepareConsume();
|
||||
case PUSH:
|
||||
return sendOrFlush();
|
||||
case FLUSH:
|
||||
return sendOrFlush();
|
||||
case FINISH:
|
||||
break; /// never reached
|
||||
}
|
||||
}
|
||||
if (status == Status::Ready)
|
||||
status = finish();
|
||||
else
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "There should be a Ready status to finish the PlanSquashing");
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
void PlanSquashingTransform::work()
|
||||
{
|
||||
prepare();
|
||||
}
|
||||
|
||||
void PlanSquashingTransform::init()
|
||||
{
|
||||
for (auto input: inputs)
|
||||
if (!input.isFinished())
|
||||
input.setNeeded();
|
||||
|
||||
planning_status = PlanningStatus::READ_IF_CAN;
|
||||
}
|
||||
|
||||
IProcessor::Status PlanSquashingTransform::prepareConsume()
|
||||
{
|
||||
bool all_finished = true;
|
||||
for (auto & input : inputs)
|
||||
{
|
||||
if (!input.isFinished())
|
||||
{
|
||||
all_finished = false;
|
||||
input.setNeeded();
|
||||
}
|
||||
else
|
||||
continue;
|
||||
|
||||
if (input.hasData())
|
||||
{
|
||||
chunk = input.pull();
|
||||
chunk = transform(std::move(chunk));
|
||||
|
||||
if (chunk.hasChunkInfo())
|
||||
{
|
||||
planning_status = PlanningStatus::PUSH;
|
||||
return Status::Ready;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (all_finished) /// If all inputs are closed, we check if we have data in balancing
|
||||
{
|
||||
if (squashing.isDataLeft()) /// If we have data in balancing, we process this data
|
||||
{
|
||||
planning_status = PlanningStatus::FLUSH;
|
||||
chunk = flushChunk();
|
||||
return Status::Ready;
|
||||
}
|
||||
planning_status = PlanningStatus::FINISH;
|
||||
return Status::Ready;
|
||||
}
|
||||
|
||||
return Status::NeedData;
|
||||
}
|
||||
|
||||
Chunk PlanSquashingTransform::transform(Chunk && chunk_)
|
||||
{
|
||||
return squashing.add(std::move(chunk_));
|
||||
}
|
||||
|
||||
Chunk PlanSquashingTransform::flushChunk()
|
||||
{
|
||||
return squashing.flush();
|
||||
}
|
||||
|
||||
IProcessor::Status PlanSquashingTransform::sendOrFlush()
|
||||
{
|
||||
if (!chunk)
|
||||
{
|
||||
planning_status = PlanningStatus::FINISH;
|
||||
return Status::Ready;
|
||||
}
|
||||
|
||||
for (auto &output : outputs)
|
||||
{
|
||||
if (output.canPush())
|
||||
{
|
||||
if (planning_status == PlanningStatus::PUSH)
|
||||
planning_status = PlanningStatus::READ_IF_CAN;
|
||||
else
|
||||
planning_status = PlanningStatus::FINISH;
|
||||
|
||||
output.push(std::move(chunk));
|
||||
return Status::Ready;
|
||||
}
|
||||
}
|
||||
return Status::PortFull;
|
||||
}
|
||||
|
||||
IProcessor::Status PlanSquashingTransform::finish()
|
||||
{
|
||||
for (auto & in : inputs)
|
||||
in.close();
|
||||
for (auto & output : outputs)
|
||||
output.finish();
|
||||
|
||||
return Status::Finished;
|
||||
}
|
||||
}
|
||||
47
src/Processors/Transforms/PlanSquashingTransform.h
Normal file
47
src/Processors/Transforms/PlanSquashingTransform.h
Normal file
@ -0,0 +1,47 @@
|
||||
#pragma once
|
||||
|
||||
#include <Processors/Sinks/SinkToStorage.h>
|
||||
#include <Processors/IProcessor.h>
|
||||
#include <Interpreters/Squashing.h>
|
||||
|
||||
enum PlanningStatus
|
||||
{
|
||||
INIT,
|
||||
READ_IF_CAN,
|
||||
PUSH,
|
||||
FLUSH,
|
||||
FINISH
|
||||
};
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class PlanSquashingTransform : public IProcessor
|
||||
{
|
||||
public:
|
||||
PlanSquashingTransform(
|
||||
const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes, size_t num_ports);
|
||||
|
||||
String getName() const override { return "PlanSquashingTransform"; }
|
||||
|
||||
InputPorts & getInputPorts() { return inputs; }
|
||||
OutputPorts & getOutputPorts() { return outputs; }
|
||||
|
||||
Status prepare() override;
|
||||
void work() override;
|
||||
void init();
|
||||
Status prepareConsume();
|
||||
Status sendOrFlush();
|
||||
Status waitForDataIn();
|
||||
Status finish();
|
||||
|
||||
Chunk transform(Chunk && chunk);
|
||||
Chunk flushChunk();
|
||||
|
||||
private:
|
||||
Chunk chunk;
|
||||
Squashing squashing;
|
||||
PlanningStatus planning_status = PlanningStatus::INIT;
|
||||
};
|
||||
}
|
||||
|
||||
@ -1,94 +0,0 @@
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
SquashingChunksTransform::SquashingChunksTransform(
|
||||
const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes)
|
||||
: ExceptionKeepingTransform(header, header, false)
|
||||
, squashing(min_block_size_rows, min_block_size_bytes)
|
||||
{
|
||||
}
|
||||
|
||||
void SquashingChunksTransform::onConsume(Chunk chunk)
|
||||
{
|
||||
if (auto block = squashing.add(getInputPort().getHeader().cloneWithColumns(chunk.detachColumns())))
|
||||
{
|
||||
cur_chunk.setColumns(block.getColumns(), block.rows());
|
||||
}
|
||||
}
|
||||
|
||||
SquashingChunksTransform::GenerateResult SquashingChunksTransform::onGenerate()
|
||||
{
|
||||
GenerateResult res;
|
||||
res.chunk = std::move(cur_chunk);
|
||||
res.is_done = true;
|
||||
return res;
|
||||
}
|
||||
|
||||
void SquashingChunksTransform::onFinish()
|
||||
{
|
||||
auto block = squashing.add({});
|
||||
finish_chunk.setColumns(block.getColumns(), block.rows());
|
||||
}
|
||||
|
||||
void SquashingChunksTransform::work()
|
||||
{
|
||||
if (stage == Stage::Exception)
|
||||
{
|
||||
data.chunk.clear();
|
||||
ready_input = false;
|
||||
return;
|
||||
}
|
||||
|
||||
ExceptionKeepingTransform::work();
|
||||
if (finish_chunk)
|
||||
{
|
||||
data.chunk = std::move(finish_chunk);
|
||||
ready_output = true;
|
||||
}
|
||||
}
|
||||
|
||||
SimpleSquashingChunksTransform::SimpleSquashingChunksTransform(
|
||||
const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes)
|
||||
: IInflatingTransform(header, header), squashing(min_block_size_rows, min_block_size_bytes)
|
||||
{
|
||||
}
|
||||
|
||||
void SimpleSquashingChunksTransform::consume(Chunk chunk)
|
||||
{
|
||||
Block current_block = squashing.add(getInputPort().getHeader().cloneWithColumns(chunk.detachColumns()));
|
||||
squashed_chunk.setColumns(current_block.getColumns(), current_block.rows());
|
||||
}
|
||||
|
||||
Chunk SimpleSquashingChunksTransform::generate()
|
||||
{
|
||||
if (squashed_chunk.empty())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't generate chunk in SimpleSquashingChunksTransform");
|
||||
|
||||
Chunk result_chunk;
|
||||
result_chunk.swap(squashed_chunk);
|
||||
return result_chunk;
|
||||
}
|
||||
|
||||
bool SimpleSquashingChunksTransform::canGenerate()
|
||||
{
|
||||
return !squashed_chunk.empty();
|
||||
}
|
||||
|
||||
Chunk SimpleSquashingChunksTransform::getRemaining()
|
||||
{
|
||||
Block current_block = squashing.add({});
|
||||
squashed_chunk.setColumns(current_block.getColumns(), current_block.rows());
|
||||
|
||||
Chunk result_chunk;
|
||||
result_chunk.swap(squashed_chunk);
|
||||
return result_chunk;
|
||||
}
|
||||
|
||||
}
|
||||
108
src/Processors/Transforms/SquashingTransform.cpp
Normal file
108
src/Processors/Transforms/SquashingTransform.cpp
Normal file
@ -0,0 +1,108 @@
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
#include <Interpreters/Squashing.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
SquashingTransform::SquashingTransform(
|
||||
const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes)
|
||||
: ExceptionKeepingTransform(header, header, false)
|
||||
, squashing(header, min_block_size_rows, min_block_size_bytes)
|
||||
{
|
||||
}
|
||||
|
||||
void SquashingTransform::onConsume(Chunk chunk)
|
||||
{
|
||||
Chunk planned_chunk = squashing.add(std::move(chunk));
|
||||
if (planned_chunk.hasChunkInfo())
|
||||
cur_chunk = DB::Squashing::squash(std::move(planned_chunk));
|
||||
}
|
||||
|
||||
SquashingTransform::GenerateResult SquashingTransform::onGenerate()
|
||||
{
|
||||
GenerateResult res;
|
||||
res.chunk = std::move(cur_chunk);
|
||||
res.is_done = true;
|
||||
return res;
|
||||
}
|
||||
|
||||
void SquashingTransform::onFinish()
|
||||
{
|
||||
Chunk chunk = squashing.flush();
|
||||
if (chunk.hasChunkInfo())
|
||||
chunk = DB::Squashing::squash(std::move(chunk));
|
||||
finish_chunk.setColumns(chunk.getColumns(), chunk.getNumRows());
|
||||
}
|
||||
|
||||
void SquashingTransform::work()
|
||||
{
|
||||
if (stage == Stage::Exception)
|
||||
{
|
||||
data.chunk.clear();
|
||||
ready_input = false;
|
||||
return;
|
||||
}
|
||||
|
||||
ExceptionKeepingTransform::work();
|
||||
if (finish_chunk)
|
||||
{
|
||||
data.chunk = std::move(finish_chunk);
|
||||
ready_output = true;
|
||||
}
|
||||
}
|
||||
|
||||
SimpleSquashingTransform::SimpleSquashingTransform(
|
||||
const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes)
|
||||
: ISimpleTransform(header, header, false)
|
||||
, squashing(header, min_block_size_rows, min_block_size_bytes)
|
||||
{
|
||||
}
|
||||
|
||||
void SimpleSquashingTransform::transform(Chunk & chunk)
|
||||
{
|
||||
if (!finished)
|
||||
{
|
||||
Chunk planned_chunk = squashing.add(std::move(chunk));
|
||||
if (planned_chunk.hasChunkInfo())
|
||||
chunk = DB::Squashing::squash(std::move(planned_chunk));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (chunk.hasRows())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk expected to be empty, otherwise it will be lost");
|
||||
|
||||
chunk = squashing.flush();
|
||||
if (chunk.hasChunkInfo())
|
||||
chunk = DB::Squashing::squash(std::move(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
IProcessor::Status SimpleSquashingTransform::prepare()
|
||||
{
|
||||
if (!finished && input.isFinished())
|
||||
{
|
||||
if (output.isFinished())
|
||||
return Status::Finished;
|
||||
|
||||
if (!output.canPush())
|
||||
return Status::PortFull;
|
||||
|
||||
if (has_output)
|
||||
{
|
||||
output.pushData(std::move(output_data));
|
||||
has_output = false;
|
||||
return Status::PortFull;
|
||||
}
|
||||
|
||||
finished = true;
|
||||
/// On the next call to transform() we will return all data buffered in `squashing` (if any)
|
||||
return Status::Ready;
|
||||
}
|
||||
return ISimpleTransform::prepare();
|
||||
}
|
||||
}
|
||||
@ -1,17 +1,17 @@
|
||||
#pragma once
|
||||
|
||||
#include <Interpreters/SquashingTransform.h>
|
||||
#include <Interpreters/Squashing.h>
|
||||
#include <Processors/ISimpleTransform.h>
|
||||
#include <Processors/IInflatingTransform.h>
|
||||
#include <Processors/Sinks/SinkToStorage.h>
|
||||
#include <Processors/Transforms/ApplySquashingTransform.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class SquashingChunksTransform : public ExceptionKeepingTransform
|
||||
class SquashingTransform : public ExceptionKeepingTransform
|
||||
{
|
||||
public:
|
||||
explicit SquashingChunksTransform(
|
||||
explicit SquashingTransform(
|
||||
const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes);
|
||||
|
||||
String getName() const override { return "SquashingTransform"; }
|
||||
@ -24,28 +24,27 @@ protected:
|
||||
void onFinish() override;
|
||||
|
||||
private:
|
||||
SquashingTransform squashing;
|
||||
Squashing squashing;
|
||||
Chunk cur_chunk;
|
||||
Chunk finish_chunk;
|
||||
};
|
||||
|
||||
/// Doesn't care about propagating exceptions and thus doesn't throw LOGICAL_ERROR if the following transform closes its input port.
|
||||
class SimpleSquashingChunksTransform : public IInflatingTransform
|
||||
class SimpleSquashingTransform : public ISimpleTransform
|
||||
{
|
||||
public:
|
||||
explicit SimpleSquashingChunksTransform(const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes);
|
||||
explicit SimpleSquashingTransform(const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes);
|
||||
|
||||
String getName() const override { return "SimpleSquashingTransform"; }
|
||||
|
||||
protected:
|
||||
void consume(Chunk chunk) override;
|
||||
bool canGenerate() override;
|
||||
Chunk generate() override;
|
||||
Chunk getRemaining() override;
|
||||
void transform(Chunk &) override;
|
||||
|
||||
IProcessor::Status prepare() override;
|
||||
|
||||
private:
|
||||
SquashingTransform squashing;
|
||||
Chunk squashed_chunk;
|
||||
};
|
||||
Squashing squashing;
|
||||
|
||||
bool finished = false;
|
||||
};
|
||||
}
|
||||
@ -6,7 +6,8 @@
|
||||
#include <Interpreters/InterpreterSelectQueryAnalyzer.h>
|
||||
#include <Parsers/ASTInsertQuery.h>
|
||||
#include <Processors/Transforms/CountingTransform.h>
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
#include <Processors/Transforms/PlanSquashingTransform.h>
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
#include <Processors/Transforms/ExpressionTransform.h>
|
||||
#include <Processors/Executors/PullingPipelineExecutor.h>
|
||||
#include <Storages/LiveView/StorageLiveView.h>
|
||||
@ -371,7 +372,7 @@ std::optional<Chain> generateViewChain(
|
||||
bool table_prefers_large_blocks = inner_table->prefersLargeBlocks();
|
||||
const auto & settings = insert_context->getSettingsRef();
|
||||
|
||||
out.addSource(std::make_shared<SquashingChunksTransform>(
|
||||
out.addSource(std::make_shared<SquashingTransform>(
|
||||
out.getInputHeader(),
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
|
||||
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL));
|
||||
@ -622,7 +623,7 @@ static QueryPipeline process(Block block, ViewRuntimeData & view, const ViewsDat
|
||||
/// Squashing is needed here because the materialized view query can generate a lot of blocks
|
||||
/// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
|
||||
/// and two-level aggregation is triggered).
|
||||
pipeline.addTransform(std::make_shared<SquashingChunksTransform>(
|
||||
pipeline.addTransform(std::make_shared<SquashingTransform>(
|
||||
pipeline.getHeader(),
|
||||
context->getSettingsRef().min_insert_block_size_rows,
|
||||
context->getSettingsRef().min_insert_block_size_bytes));
|
||||
|
||||
@ -1063,7 +1063,6 @@ void HTTPHandler::formatExceptionForClient(int exception_code, HTTPServerRequest
|
||||
void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & write_event)
|
||||
{
|
||||
setThreadName("HTTPHandler");
|
||||
ThreadStatus thread_status;
|
||||
|
||||
session = std::make_unique<Session>(server.context(), ClientInfo::Interface::HTTP, request.isSecure());
|
||||
SCOPE_EXIT({ session.reset(); });
|
||||
|
||||
@ -8,7 +8,6 @@
|
||||
#include <Interpreters/InterserverIOHandler.h>
|
||||
#include <Server/HTTP/HTMLForm.h>
|
||||
#include <Server/HTTP/WriteBufferFromHTTPServerResponse.h>
|
||||
#include <Common/ThreadStatus.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/setThreadName.h>
|
||||
|
||||
@ -81,7 +80,6 @@ void InterserverIOHTTPHandler::processQuery(HTTPServerRequest & request, HTTPSer
|
||||
void InterserverIOHTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & write_event)
|
||||
{
|
||||
setThreadName("IntersrvHandler");
|
||||
ThreadStatus thread_status;
|
||||
|
||||
/// In order to work keep-alive.
|
||||
if (request.getVersion() == HTTPServerRequest::HTTP_1_1)
|
||||
|
||||
@ -309,7 +309,6 @@ Poco::Timespan KeeperTCPHandler::receiveHandshake(int32_t handshake_length, bool
|
||||
void KeeperTCPHandler::runImpl()
|
||||
{
|
||||
setThreadName("KeeperHandler");
|
||||
ThreadStatus thread_status;
|
||||
|
||||
socket().setReceiveTimeout(receive_timeout);
|
||||
socket().setSendTimeout(send_timeout);
|
||||
|
||||
@ -24,7 +24,6 @@
|
||||
#include <Common/CurrentThread.h>
|
||||
#include <Common/NetException.h>
|
||||
#include <Common/OpenSSLHelpers.h>
|
||||
#include <Common/ThreadStatus.h>
|
||||
#include <Common/config_version.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/re2.h>
|
||||
@ -199,7 +198,6 @@ MySQLHandler::~MySQLHandler() = default;
|
||||
void MySQLHandler::run()
|
||||
{
|
||||
setThreadName("MySQLHandler");
|
||||
ThreadStatus thread_status;
|
||||
|
||||
session = std::make_unique<Session>(server.context(), ClientInfo::Interface::MYSQL);
|
||||
SCOPE_EXIT({ session.reset(); });
|
||||
|
||||
@ -10,7 +10,6 @@
|
||||
#include <base/scope_guard.h>
|
||||
#include <pcg_random.hpp>
|
||||
#include <Common/CurrentThread.h>
|
||||
#include <Common/ThreadStatus.h>
|
||||
#include <Common/config_version.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <Common/setThreadName.h>
|
||||
@ -59,7 +58,6 @@ void PostgreSQLHandler::changeIO(Poco::Net::StreamSocket & socket)
|
||||
void PostgreSQLHandler::run()
|
||||
{
|
||||
setThreadName("PostgresHandler");
|
||||
ThreadStatus thread_status;
|
||||
|
||||
session = std::make_unique<Session>(server.context(), ClientInfo::Interface::POSTGRESQL);
|
||||
SCOPE_EXIT({ session.reset(); });
|
||||
|
||||
@ -1,9 +1,8 @@
|
||||
#include "Interpreters/AsynchronousInsertQueue.h"
|
||||
#include "Interpreters/SquashingTransform.h"
|
||||
#include "Parsers/ASTInsertQuery.h"
|
||||
#include <Interpreters/AsynchronousInsertQueue.h>
|
||||
#include <Interpreters/Squashing.h>
|
||||
#include <Parsers/ASTInsertQuery.h>
|
||||
#include <algorithm>
|
||||
#include <exception>
|
||||
#include <iterator>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
@ -246,7 +245,6 @@ TCPHandler::~TCPHandler()
|
||||
void TCPHandler::runImpl()
|
||||
{
|
||||
setThreadName("TCPHandler");
|
||||
ThreadStatus thread_status;
|
||||
|
||||
extractConnectionSettingsFromContext(server.context());
|
||||
|
||||
@ -886,13 +884,16 @@ AsynchronousInsertQueue::PushResult TCPHandler::processAsyncInsertQuery(Asynchro
|
||||
using PushResult = AsynchronousInsertQueue::PushResult;
|
||||
|
||||
startInsertQuery();
|
||||
SquashingTransform squashing(0, query_context->getSettingsRef().async_insert_max_data_size);
|
||||
Squashing squashing(state.input_header, 0, query_context->getSettingsRef().async_insert_max_data_size);
|
||||
|
||||
while (readDataNext())
|
||||
{
|
||||
auto result = squashing.add(std::move(state.block_for_insert));
|
||||
if (result)
|
||||
squashing.header = state.block_for_insert;
|
||||
auto planned_chunk = squashing.add({state.block_for_insert.getColumns(), state.block_for_insert.rows()});
|
||||
if (planned_chunk.hasChunkInfo())
|
||||
{
|
||||
Chunk result_chunk = DB::Squashing::squash(std::move(planned_chunk));
|
||||
auto result = state.block_for_insert.cloneWithColumns(result_chunk.getColumns());
|
||||
return PushResult
|
||||
{
|
||||
.status = PushResult::TOO_MUCH_DATA,
|
||||
@ -901,7 +902,12 @@ AsynchronousInsertQueue::PushResult TCPHandler::processAsyncInsertQuery(Asynchro
|
||||
}
|
||||
}
|
||||
|
||||
auto result = squashing.add({});
|
||||
auto planned_chunk = squashing.flush();
|
||||
Chunk result_chunk;
|
||||
if (planned_chunk.hasChunkInfo())
|
||||
result_chunk = DB::Squashing::squash(std::move(planned_chunk));
|
||||
|
||||
auto result = squashing.header.cloneWithColumns(result_chunk.getColumns());
|
||||
return insert_queue.pushQueryWithBlock(state.parsed_query, std::move(result), query_context);
|
||||
}
|
||||
|
||||
|
||||
@ -7,7 +7,6 @@
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/ThreadStatus.h>
|
||||
#include <Core/Protocol.h>
|
||||
#include <Core/QueryProcessingStage.h>
|
||||
#include <IO/Progress.h>
|
||||
|
||||
@ -21,7 +21,7 @@ limitations under the License. */
|
||||
#include <Processors/Transforms/MaterializingTransform.h>
|
||||
#include <Processors/Executors/PullingAsyncPipelineExecutor.h>
|
||||
#include <Processors/Executors/PipelineExecutor.h>
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
#include <QueryPipeline/QueryPipelineBuilder.h>
|
||||
#include <QueryPipeline/QueryPlanResourceHolder.h>
|
||||
#include <Common/logger_useful.h>
|
||||
@ -626,7 +626,7 @@ QueryPipelineBuilder StorageLiveView::completeQuery(Pipes pipes)
|
||||
/// and two-level aggregation is triggered).
|
||||
builder.addSimpleTransform([&](const Block & cur_header)
|
||||
{
|
||||
return std::make_shared<SquashingChunksTransform>(
|
||||
return std::make_shared<SquashingTransform>(
|
||||
cur_header,
|
||||
getContext()->getSettingsRef().min_insert_block_size_rows,
|
||||
getContext()->getSettingsRef().min_insert_block_size_bytes);
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
#include <Storages/Statistics/Statistics.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
#include <Interpreters/SquashingTransform.h>
|
||||
#include <Interpreters/Squashing.h>
|
||||
#include <Interpreters/MergeTreeTransaction.h>
|
||||
#include <Interpreters/PreparedSets.h>
|
||||
#include <Processors/Transforms/TTLTransform.h>
|
||||
@ -29,6 +29,7 @@
|
||||
#include <DataTypes/DataTypeVariant.h>
|
||||
#include <boost/algorithm/string/replace.hpp>
|
||||
#include <Common/ProfileEventsScope.h>
|
||||
#include <Core/ColumnsWithTypeAndName.h>
|
||||
|
||||
|
||||
namespace ProfileEvents
|
||||
@ -1267,7 +1268,7 @@ private:
|
||||
ProjectionNameToItsBlocks projection_parts;
|
||||
std::move_iterator<ProjectionNameToItsBlocks::iterator> projection_parts_iterator;
|
||||
|
||||
std::vector<SquashingTransform> projection_squashes;
|
||||
std::vector<Squashing> projection_squashes;
|
||||
const ProjectionsDescription & projections;
|
||||
|
||||
ExecutableTaskPtr merge_projection_parts_task_ptr;
|
||||
@ -1286,7 +1287,7 @@ void PartMergerWriter::prepare()
|
||||
for (size_t i = 0, size = ctx->projections_to_build.size(); i < size; ++i)
|
||||
{
|
||||
// We split the materialization into multiple stages similar to the process of INSERT SELECT query.
|
||||
projection_squashes.emplace_back(settings.min_insert_block_size_rows, settings.min_insert_block_size_bytes);
|
||||
projection_squashes.emplace_back(ctx->updated_header, settings.min_insert_block_size_rows, settings.min_insert_block_size_bytes);
|
||||
}
|
||||
|
||||
existing_rows_count = 0;
|
||||
@ -1311,16 +1312,18 @@ bool PartMergerWriter::mutateOriginalPartAndPrepareProjections()
|
||||
{
|
||||
const auto & projection = *ctx->projections_to_build[i];
|
||||
|
||||
Block projection_block;
|
||||
{
|
||||
ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::MutateTaskProjectionsCalculationMicroseconds);
|
||||
projection_block = projection_squashes[i].add(projection.calculate(cur_block, ctx->context));
|
||||
}
|
||||
ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::MutateTaskProjectionsCalculationMicroseconds);
|
||||
Block block_to_squash = projection.calculate(cur_block, ctx->context);
|
||||
projection_squashes[i].header = block_to_squash;
|
||||
Chunk planned_chunk = projection_squashes[i].add({block_to_squash.getColumns(), block_to_squash.rows()});
|
||||
|
||||
if (projection_block)
|
||||
if (planned_chunk.hasChunkInfo())
|
||||
{
|
||||
Chunk projection_chunk = DB::Squashing::squash(std::move(planned_chunk));
|
||||
|
||||
auto result = block_to_squash.cloneWithColumns(projection_chunk.getColumns());
|
||||
auto tmp_part = MergeTreeDataWriter::writeTempProjectionPart(
|
||||
*ctx->data, ctx->log, projection_block, projection, ctx->new_data_part.get(), ++block_num);
|
||||
*ctx->data, ctx->log, result, projection, ctx->new_data_part.get(), ++block_num);
|
||||
tmp_part.finalize();
|
||||
tmp_part.part->getDataPartStorage().commitTransaction();
|
||||
projection_parts[projection.name].emplace_back(std::move(tmp_part.part));
|
||||
@ -1338,12 +1341,15 @@ bool PartMergerWriter::mutateOriginalPartAndPrepareProjections()
|
||||
for (size_t i = 0, size = ctx->projections_to_build.size(); i < size; ++i)
|
||||
{
|
||||
const auto & projection = *ctx->projections_to_build[i];
|
||||
auto & projection_squash = projection_squashes[i];
|
||||
auto projection_block = projection_squash.add({});
|
||||
if (projection_block)
|
||||
auto & projection_squash_plan = projection_squashes[i];
|
||||
auto planned_chunk = projection_squash_plan.flush();
|
||||
if (planned_chunk.hasChunkInfo())
|
||||
{
|
||||
Chunk projection_chunk = DB::Squashing::squash(std::move(planned_chunk));
|
||||
|
||||
auto result = projection_squash_plan.header.cloneWithColumns(projection_chunk.getColumns());
|
||||
auto temp_part = MergeTreeDataWriter::writeTempProjectionPart(
|
||||
*ctx->data, ctx->log, projection_block, projection, ctx->new_data_part.get(), ++block_num);
|
||||
*ctx->data, ctx->log, result, projection, ctx->new_data_part.get(), ++block_num);
|
||||
temp_part.finalize();
|
||||
temp_part.part->getDataPartStorage().commitTransaction();
|
||||
projection_parts[projection.name].emplace_back(std::move(temp_part.part));
|
||||
|
||||
@ -16,7 +16,8 @@
|
||||
#include <Processors/Executors/PullingPipelineExecutor.h>
|
||||
#include <Processors/QueryPlan/QueryPlan.h>
|
||||
#include <Processors/Sources/SourceFromSingleChunk.h>
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
#include <Processors/Transforms/PlanSquashingTransform.h>
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
#include <QueryPipeline/Pipe.h>
|
||||
#include <QueryPipeline/QueryPipelineBuilder.h>
|
||||
#include <base/range.h>
|
||||
@ -306,7 +307,9 @@ Block ProjectionDescription::calculate(const Block & block, ContextPtr context)
|
||||
builder.resize(1);
|
||||
// Generate aggregated blocks with rows less or equal than the original block.
|
||||
// There should be only one output block after this transformation.
|
||||
builder.addTransform(std::make_shared<SquashingChunksTransform>(builder.getHeader(), block.rows(), 0));
|
||||
|
||||
builder.addTransform(std::make_shared<PlanSquashingTransform>(builder.getHeader(), block.rows(), 0, 1));
|
||||
builder.addTransform(std::make_shared<ApplySquashingTransform>(builder.getHeader(), block.rows(), 0));
|
||||
|
||||
auto pipeline = QueryPipelineBuilder::getPipeline(std::move(builder));
|
||||
PullingPipelineExecutor executor(pipeline);
|
||||
|
||||
@ -298,7 +298,7 @@ private:
|
||||
ClientInfo::Interface client_info_interface;
|
||||
size_t db_table_num = 0;
|
||||
size_t total_tables;
|
||||
std::shared_ptr<const ContextAccess> access;
|
||||
std::shared_ptr<const ContextAccessWrapper> access;
|
||||
bool need_to_check_access_for_tables;
|
||||
String query_id;
|
||||
std::chrono::milliseconds lock_acquire_timeout;
|
||||
|
||||
@ -35,7 +35,7 @@
|
||||
#include <Processors/Transforms/ExpressionTransform.h>
|
||||
#include <Processors/Transforms/FilterTransform.h>
|
||||
#include <Processors/Transforms/WatermarkTransform.h>
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
#include <Processors/Transforms/MaterializingTransform.h>
|
||||
#include <Processors/QueryPlan/QueryPlan.h>
|
||||
#include <Processors/QueryPlan/BuildQueryPipelineSettings.h>
|
||||
@ -633,7 +633,7 @@ std::pair<BlocksPtr, Block> StorageWindowView::getNewBlocks(UInt32 watermark)
|
||||
});
|
||||
builder.addSimpleTransform([&](const Block & current_header)
|
||||
{
|
||||
return std::make_shared<SquashingChunksTransform>(
|
||||
return std::make_shared<SquashingTransform>(
|
||||
current_header,
|
||||
getContext()->getSettingsRef().min_insert_block_size_rows,
|
||||
getContext()->getSettingsRef().min_insert_block_size_bytes);
|
||||
@ -1532,7 +1532,7 @@ void StorageWindowView::writeIntoWindowView(
|
||||
builder = select_block.buildQueryPipeline();
|
||||
builder.addSimpleTransform([&](const Block & current_header)
|
||||
{
|
||||
return std::make_shared<SquashingChunksTransform>(
|
||||
return std::make_shared<SquashingTransform>(
|
||||
current_header,
|
||||
local_context->getSettingsRef().min_insert_block_size_rows,
|
||||
local_context->getSettingsRef().min_insert_block_size_bytes);
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
#include <Processors/Executors/CompletedPipelineExecutor.h>
|
||||
#include <Processors/QueryPlan/ExpressionStep.h>
|
||||
#include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
|
||||
#include <Processors/Transforms/SquashingChunksTransform.h>
|
||||
#include <Processors/Transforms/SquashingTransform.h>
|
||||
#include <QueryPipeline/QueryPipelineBuilder.h>
|
||||
#include <Storages/removeGroupingFunctionSpecializations.h>
|
||||
#include <Storages/StorageDistributed.h>
|
||||
@ -290,7 +290,7 @@ TableNodePtr executeSubqueryNode(const QueryTreeNodePtr & subquery_node,
|
||||
|
||||
size_t min_block_size_rows = mutable_context->getSettingsRef().min_external_table_block_size_rows;
|
||||
size_t min_block_size_bytes = mutable_context->getSettingsRef().min_external_table_block_size_bytes;
|
||||
auto squashing = std::make_shared<SimpleSquashingChunksTransform>(builder->getHeader(), min_block_size_rows, min_block_size_bytes);
|
||||
auto squashing = std::make_shared<SimpleSquashingTransform>(builder->getHeader(), min_block_size_rows, min_block_size_bytes);
|
||||
|
||||
builder->resize(1);
|
||||
builder->addTransform(std::move(squashing));
|
||||
|
||||
@ -300,7 +300,7 @@ def test_when_s3_broken_pipe_at_upload_is_retried(cluster, broken_s3):
|
||||
LIMIT 1000000
|
||||
SETTINGS
|
||||
s3_max_single_part_upload_size=100,
|
||||
s3_min_upload_part_size=1000000,
|
||||
s3_min_upload_part_size=100000,
|
||||
s3_check_objects_after_upload=0
|
||||
""",
|
||||
query_id=insert_query_id,
|
||||
@ -311,7 +311,7 @@ def test_when_s3_broken_pipe_at_upload_is_retried(cluster, broken_s3):
|
||||
)
|
||||
|
||||
assert create_multipart == 1
|
||||
assert upload_parts == 7
|
||||
assert upload_parts == 69
|
||||
assert s3_errors == 3
|
||||
|
||||
broken_s3.setup_at_part_upload(
|
||||
|
||||
0
tests/integration/test_keeper_profiler/__init__.py
Normal file
0
tests/integration/test_keeper_profiler/__init__.py
Normal file
@ -0,0 +1,44 @@
|
||||
<clickhouse>
|
||||
<keeper_server>
|
||||
<s3_snapshot>
|
||||
<endpoint>http://minio1:9001/snapshots/</endpoint>
|
||||
<access_key_id>minio</access_key_id>
|
||||
<secret_access_key>minio123</secret_access_key>
|
||||
</s3_snapshot>
|
||||
<tcp_port>9181</tcp_port>
|
||||
<server_id>1</server_id>
|
||||
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
||||
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
||||
<four_letter_word_white_list>*</four_letter_word_white_list>
|
||||
|
||||
<coordination_settings>
|
||||
<operation_timeout_ms>5000</operation_timeout_ms>
|
||||
<session_timeout_ms>10000</session_timeout_ms>
|
||||
<min_session_timeout_ms>5000</min_session_timeout_ms>
|
||||
<snapshot_distance>50</snapshot_distance>
|
||||
<raft_logs_level>trace</raft_logs_level>
|
||||
</coordination_settings>
|
||||
|
||||
<raft_configuration>
|
||||
<server>
|
||||
<id>1</id>
|
||||
<hostname>node1</hostname>
|
||||
<port>9234</port>
|
||||
</server>
|
||||
<server>
|
||||
<id>2</id>
|
||||
<hostname>node2</hostname>
|
||||
<port>9234</port>
|
||||
<start_as_follower>true</start_as_follower>
|
||||
</server>
|
||||
<server>
|
||||
<id>3</id>
|
||||
<hostname>node3</hostname>
|
||||
<port>9234</port>
|
||||
<start_as_follower>true</start_as_follower>
|
||||
</server>
|
||||
</raft_configuration>
|
||||
</keeper_server>
|
||||
<global_profiler_real_time_period_ns>1000000000</global_profiler_real_time_period_ns>
|
||||
<global_profiler_cpu_time_period_ns>1000000000</global_profiler_cpu_time_period_ns>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,44 @@
|
||||
<clickhouse>
|
||||
<keeper_server>
|
||||
<s3_snapshot>
|
||||
<endpoint>http://minio1:9001/snapshots/</endpoint>
|
||||
<access_key_id>minio</access_key_id>
|
||||
<secret_access_key>minio123</secret_access_key>
|
||||
</s3_snapshot>
|
||||
<tcp_port>9181</tcp_port>
|
||||
<server_id>2</server_id>
|
||||
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
||||
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
||||
<four_letter_word_white_list>*</four_letter_word_white_list>
|
||||
|
||||
<coordination_settings>
|
||||
<operation_timeout_ms>5000</operation_timeout_ms>
|
||||
<session_timeout_ms>10000</session_timeout_ms>
|
||||
<min_session_timeout_ms>5000</min_session_timeout_ms>
|
||||
<snapshot_distance>75</snapshot_distance>
|
||||
<raft_logs_level>trace</raft_logs_level>
|
||||
</coordination_settings>
|
||||
|
||||
<raft_configuration>
|
||||
<server>
|
||||
<id>1</id>
|
||||
<hostname>node1</hostname>
|
||||
<port>9234</port>
|
||||
</server>
|
||||
<server>
|
||||
<id>2</id>
|
||||
<hostname>node2</hostname>
|
||||
<port>9234</port>
|
||||
<start_as_follower>true</start_as_follower>
|
||||
</server>
|
||||
<server>
|
||||
<id>3</id>
|
||||
<hostname>node3</hostname>
|
||||
<port>9234</port>
|
||||
<start_as_follower>true</start_as_follower>
|
||||
</server>
|
||||
</raft_configuration>
|
||||
</keeper_server>
|
||||
<global_profiler_real_time_period_ns>1000000000</global_profiler_real_time_period_ns>
|
||||
<global_profiler_cpu_time_period_ns>1000000000</global_profiler_cpu_time_period_ns>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,44 @@
|
||||
<clickhouse>
|
||||
<keeper_server>
|
||||
<s3_snapshot>
|
||||
<endpoint>http://minio1:9001/snapshots/</endpoint>
|
||||
<access_key_id>minio</access_key_id>
|
||||
<secret_access_key>minio123</secret_access_key>
|
||||
</s3_snapshot>
|
||||
<tcp_port>9181</tcp_port>
|
||||
<server_id>3</server_id>
|
||||
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
||||
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
||||
<four_letter_word_white_list>*</four_letter_word_white_list>
|
||||
|
||||
<coordination_settings>
|
||||
<operation_timeout_ms>5000</operation_timeout_ms>
|
||||
<session_timeout_ms>10000</session_timeout_ms>
|
||||
<min_session_timeout_ms>5000</min_session_timeout_ms>
|
||||
<snapshot_distance>75</snapshot_distance>
|
||||
<raft_logs_level>trace</raft_logs_level>
|
||||
</coordination_settings>
|
||||
|
||||
<raft_configuration>
|
||||
<server>
|
||||
<id>1</id>
|
||||
<hostname>node1</hostname>
|
||||
<port>9234</port>
|
||||
</server>
|
||||
<server>
|
||||
<id>2</id>
|
||||
<hostname>node2</hostname>
|
||||
<port>9234</port>
|
||||
<start_as_follower>true</start_as_follower>
|
||||
</server>
|
||||
<server>
|
||||
<id>3</id>
|
||||
<hostname>node3</hostname>
|
||||
<port>9234</port>
|
||||
<start_as_follower>true</start_as_follower>
|
||||
</server>
|
||||
</raft_configuration>
|
||||
</keeper_server>
|
||||
<global_profiler_real_time_period_ns>1000000000</global_profiler_real_time_period_ns>
|
||||
<global_profiler_cpu_time_period_ns>1000000000</global_profiler_cpu_time_period_ns>
|
||||
</clickhouse>
|
||||
96
tests/integration/test_keeper_profiler/test.py
Normal file
96
tests/integration/test_keeper_profiler/test.py
Normal file
@ -0,0 +1,96 @@
|
||||
import pytest
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.test_tools import TSV
|
||||
from helpers.keeper_utils import KeeperClient, KeeperException
|
||||
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
node = cluster.add_instance(
|
||||
"node1",
|
||||
main_configs=["configs/keeper_config1.xml"],
|
||||
stay_alive=True,
|
||||
)
|
||||
node2 = cluster.add_instance(
|
||||
"node2",
|
||||
main_configs=["configs/keeper_config2.xml"],
|
||||
stay_alive=True,
|
||||
with_minio=True,
|
||||
)
|
||||
node3 = cluster.add_instance(
|
||||
"node3",
|
||||
main_configs=["configs/keeper_config3.xml"],
|
||||
stay_alive=True,
|
||||
with_minio=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def test_profiler(started_cluster):
|
||||
node = cluster.instances["node1"]
|
||||
if node.is_built_with_sanitizer():
|
||||
return
|
||||
|
||||
node.query(
|
||||
"CREATE TABLE t (key UInt32, value String) Engine = ReplicatedMergeTree('/clickhouse-tables/test1', 'r1') ORDER BY key"
|
||||
)
|
||||
|
||||
for _ in range(100):
|
||||
node.query("INSERT INTO t SELECT number, toString(number) from numbers(100)")
|
||||
|
||||
node.query("system flush logs")
|
||||
assert int(node.query("exists system.trace_log"))
|
||||
|
||||
result = node.query(
|
||||
"""
|
||||
set allow_introspection_functions=1;
|
||||
system flush logs;
|
||||
select cnt from (
|
||||
select count() as cnt, formatReadableSize(sum(size)),
|
||||
arrayStringConcat(
|
||||
arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), arrayMap(x -> demangle(addressToSymbol(x)), trace)),
|
||||
'\n') as trace
|
||||
from system.trace_log where trace_type = ‘Real’ and (trace ilike '%KeeperTCPHandler%' or trace ilike '%KeeperDispatcher%') group by trace order by cnt desc) limit 1;
|
||||
"""
|
||||
)
|
||||
|
||||
if len(result) == 0:
|
||||
assert 0 < int(
|
||||
node.query(
|
||||
"""
|
||||
set allow_introspection_functions=1;
|
||||
system flush logs;
|
||||
select sum(cnt) from (
|
||||
select count() as cnt, formatReadableSize(sum(size)),
|
||||
arrayStringConcat(
|
||||
arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), arrayMap(x -> demangle(addressToSymbol(x)), trace)),
|
||||
'\n') as trace
|
||||
from system.trace_log where trace_type = ‘Real’ group by trace);
|
||||
"""
|
||||
)
|
||||
)
|
||||
result = node.query(
|
||||
"""
|
||||
set allow_introspection_functions=1;
|
||||
system flush logs;
|
||||
select * from (
|
||||
select count() as cnt, formatReadableSize(sum(size)),
|
||||
arrayStringConcat(
|
||||
arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), arrayMap(x -> demangle(addressToSymbol(x)), trace)),
|
||||
'\n') as trace
|
||||
from system.trace_log where trace_type = ‘Real’ group by trace);
|
||||
"""
|
||||
)
|
||||
print(result)
|
||||
assert False
|
||||
|
||||
assert 1 < int(result)
|
||||
@ -0,0 +1,5 @@
|
||||
<clickhouse>
|
||||
<max_table_num_to_throw>10</max_table_num_to_throw>
|
||||
<max_database_num_to_throw>10</max_database_num_to_throw>
|
||||
</clickhouse>
|
||||
|
||||
43
tests/integration/test_table_db_num_limit/test.py
Normal file
43
tests/integration/test_table_db_num_limit/test.py
Normal file
@ -0,0 +1,43 @@
|
||||
import pytest
|
||||
from helpers.client import QueryRuntimeException
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
node1 = cluster.add_instance(
|
||||
"node1", main_configs=["config/config.xml"], with_zookeeper=True
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def test_table_db_limit(started_cluster):
|
||||
for i in range(10):
|
||||
node1.query("create database db{}".format(i))
|
||||
|
||||
with pytest.raises(QueryRuntimeException) as exp_info:
|
||||
node1.query("create database db_exp".format(i))
|
||||
|
||||
assert "TOO_MANY_DATABASES" in str(exp_info)
|
||||
|
||||
for i in range(10):
|
||||
node1.query("create table t{} (a Int32) Engine = Log".format(i))
|
||||
|
||||
node1.query("system flush logs")
|
||||
for i in range(10):
|
||||
node1.query("drop table t{}".format(i))
|
||||
for i in range(10):
|
||||
node1.query("create table t{} (a Int32) Engine = Log".format(i))
|
||||
|
||||
with pytest.raises(QueryRuntimeException) as exp_info:
|
||||
node1.query("create table default.tx (a Int32) Engine = Log")
|
||||
assert "TOO_MANY_TABLES" in str(exp_info)
|
||||
@ -13,6 +13,7 @@
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
0
|
||||
0
|
||||
0
|
||||
|
||||
@ -13,6 +13,7 @@ SELECT 'Hello' IN (SELECT 'Hello');
|
||||
SELECT materialize('Hello') IN (SELECT 'Hello');
|
||||
SELECT 'Hello' IN (SELECT materialize('Hello'));
|
||||
SELECT materialize('Hello') IN (SELECT materialize('Hello'));
|
||||
SELECT toDate('2020-01-01') IN (toDateTime('2020-01-01', 'UTC'));
|
||||
|
||||
SELECT 2 IN (SELECT 1);
|
||||
SELECT materialize(2) IN (SELECT 1);
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
SET uniform_snowflake_conversion_functions = 0;
|
||||
|
||||
SELECT addMonths(toDateTime('2017-11-05 08:07:47', 'Asia/Istanbul'), 1, 'Asia/Kolkata');
|
||||
SELECT addMonths(toDateTime('2017-11-05 10:37:47', 'Asia/Kolkata'), 1);
|
||||
SELECT addMonths(toTimeZone(toDateTime('2017-11-05 08:07:47', 'Asia/Istanbul'), 'Asia/Kolkata'), 1);
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
SET uniform_snowflake_conversion_functions = 0; -- Force-disable uniform snowflake conversion functions (in case this is randomized in CI)
|
||||
SET session_timezone = 'Africa/Juba';
|
||||
|
||||
-- Error cases
|
||||
@ -10,6 +11,9 @@ SELECT dateTime64ToSnowflake('abc'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT dateTimeToSnowflake('abc', 123); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT dateTime64ToSnowflake('abc', 123); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
|
||||
SELECT dateTimeToSnowflake(now()) SETTINGS uniform_snowflake_conversion_functions = 1; -- { serverError DEPRECATED_FUNCTION }
|
||||
SELECT dateTime64ToSnowflake(now64()) SETTINGS uniform_snowflake_conversion_functions = 1; -- { serverError DEPRECATED_FUNCTION }
|
||||
|
||||
SELECT '-- const / non-const inputs';
|
||||
|
||||
WITH toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai') AS dt
|
||||
|
||||
@ -0,0 +1,32 @@
|
||||
-- Negative tests
|
||||
-- Return type
|
||||
UInt64
|
||||
UInt64
|
||||
-- Standard and twitter epoch
|
||||
Row 1:
|
||||
──────
|
||||
dt: 2021-08-15 18:57:56
|
||||
dt64: 2021-08-15 18:57:56.492
|
||||
dateTimeToSnowflakeID(dt): 6832747188322304000
|
||||
dateTime64ToSnowflakeID(dt64): 6832747190385901568
|
||||
dateTimeToSnowflakeID(dt, twitter_epoch): 1426981498778550272
|
||||
dateTime64ToSnowflakeID(dt64, twitter_epoch): 1426981500842147840
|
||||
-- Different DateTime64 scales
|
||||
Row 1:
|
||||
──────
|
||||
dateTime64ToSnowflakeID(dt64_0): 6832747188322304000
|
||||
dateTime64ToSnowflakeID(dt64_1): 6832747190000025600
|
||||
dateTime64ToSnowflakeID(dt64_2): 6832747190377512960
|
||||
dateTime64ToSnowflakeID(dt64_3): 6832747190385901568
|
||||
dateTime64ToSnowflakeID(dt64_4): 6832747190385901568
|
||||
-- Idempotency
|
||||
Row 1:
|
||||
──────
|
||||
equals(snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_0), 0, 'UTC'), dt64_0): 1
|
||||
equals(snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_1), 0, 'UTC'), dt64_1): 1
|
||||
equals(snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_2), 0, 'UTC'), dt64_2): 1
|
||||
equals(snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_3), 0, 'UTC'), dt64_3): 1
|
||||
Row 1:
|
||||
──────
|
||||
dt64_4: 2023-11-11 11:11:11.1231
|
||||
snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_4)): 2023-11-11 11:11:11.123
|
||||
78
tests/queries/0_stateless/01942_dateTimeToSnowflakeID.sql
Normal file
78
tests/queries/0_stateless/01942_dateTimeToSnowflakeID.sql
Normal file
@ -0,0 +1,78 @@
|
||||
SET session_timezone = 'UTC'; -- disable timezone randomization
|
||||
SET allow_experimental_analyzer = 1; -- The old path formats the result with different whitespaces
|
||||
SET uniform_snowflake_conversion_functions = 1; -- Force-enable uniform snowflake conversion functions (in case this is randomized in CI)
|
||||
|
||||
SELECT '-- Negative tests';
|
||||
SELECT dateTimeToSnowflakeID(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT dateTime64ToSnowflakeID(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT dateTimeToSnowflakeID('invalid_dt'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT dateTime64ToSnowflakeID('invalid_dt'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT dateTimeToSnowflakeID(now(), 'invalid_epoch'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT dateTime64ToSnowflakeID(now64(), 'invalid_epoch'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT dateTimeToSnowflakeID(now(), 42, 'too_many_args'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT dateTime64ToSnowflakeID(now64(), 42, 'too_many_args'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
|
||||
SELECT dateTimeToSnowflakeID(now()) SETTINGS uniform_snowflake_conversion_functions = 0; -- { serverError UNKNOWN_FUNCTION }
|
||||
SELECT dateTime64ToSnowflakeID(now64()) SETTINGS uniform_snowflake_conversion_functions = 0; -- { serverError UNKNOWN_FUNCTION }
|
||||
|
||||
SELECT '-- Return type';
|
||||
SELECT toTypeName(dateTimeToSnowflakeID(now()));
|
||||
SELECT toTypeName(dateTime64ToSnowflakeID(now64()));
|
||||
|
||||
SELECT '-- Standard and twitter epoch';
|
||||
|
||||
WITH
|
||||
toDateTime('2021-08-15 18:57:56') AS dt,
|
||||
toDateTime64('2021-08-15 18:57:56.492', 3) AS dt64,
|
||||
1288834974657 AS twitter_epoch
|
||||
SELECT
|
||||
dt,
|
||||
dt64,
|
||||
dateTimeToSnowflakeID(dt),
|
||||
dateTime64ToSnowflakeID(dt64),
|
||||
dateTimeToSnowflakeID(dt, twitter_epoch),
|
||||
dateTime64ToSnowflakeID(dt64, twitter_epoch)
|
||||
FORMAT
|
||||
Vertical;
|
||||
|
||||
SELECT '-- Different DateTime64 scales';
|
||||
|
||||
WITH
|
||||
toDateTime64('2021-08-15 18:57:56.492', 0, 'UTC') AS dt64_0,
|
||||
toDateTime64('2021-08-15 18:57:56.492', 1, 'UTC') AS dt64_1,
|
||||
toDateTime64('2021-08-15 18:57:56.492', 2, 'UTC') AS dt64_2,
|
||||
toDateTime64('2021-08-15 18:57:56.492', 3, 'UTC') AS dt64_3,
|
||||
toDateTime64('2021-08-15 18:57:56.492', 4, 'UTC') AS dt64_4
|
||||
SELECT
|
||||
dateTime64ToSnowflakeID(dt64_0),
|
||||
dateTime64ToSnowflakeID(dt64_1),
|
||||
dateTime64ToSnowflakeID(dt64_2),
|
||||
dateTime64ToSnowflakeID(dt64_3),
|
||||
dateTime64ToSnowflakeID(dt64_4)
|
||||
Format
|
||||
Vertical;
|
||||
|
||||
SELECT '-- Idempotency';
|
||||
|
||||
-- DateTime64-to-SnowflakeID-to-DateTime64 is idempotent if the scale is <=3 (millisecond precision)
|
||||
WITH
|
||||
now64(0) AS dt64_0,
|
||||
now64(1) AS dt64_1,
|
||||
now64(2) AS dt64_2,
|
||||
now64(3) AS dt64_3
|
||||
SELECT
|
||||
snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_0), 0, 'UTC') == dt64_0,
|
||||
snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_1), 0, 'UTC') == dt64_1,
|
||||
snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_2), 0, 'UTC') == dt64_2,
|
||||
snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_3), 0, 'UTC') == dt64_3
|
||||
FORMAT
|
||||
Vertical;
|
||||
|
||||
-- not idempotent
|
||||
WITH
|
||||
toDateTime64('2023-11-11 11:11:11.1231', 4, 'UTC') AS dt64_4
|
||||
SELECT
|
||||
dt64_4,
|
||||
snowflakeIDToDateTime64(dateTime64ToSnowflakeID(dt64_4))
|
||||
FORMAT
|
||||
Vertical;
|
||||
@ -0,0 +1,27 @@
|
||||
-- Negative tests
|
||||
-- Return type
|
||||
DateTime
|
||||
DateTime64(3)
|
||||
-- Non-const path
|
||||
Row 1:
|
||||
──────
|
||||
sf: 7204436857747984384
|
||||
dt: 2024-06-06 10:59:58
|
||||
dt64: 2024-06-06 10:59:58.851
|
||||
Row 1:
|
||||
──────
|
||||
sf: 1426981498778550272
|
||||
dt: 2021-08-15 18:57:56
|
||||
dt64: 2021-08-15 18:57:56.000
|
||||
Row 1:
|
||||
──────
|
||||
sf: 7204436857747984384
|
||||
dt: 2024-06-06 18:59:58
|
||||
dt64: 2024-06-06 18:59:58.851
|
||||
-- Const path
|
||||
Row 1:
|
||||
──────
|
||||
sf: 7204436857747984384
|
||||
dt: 2024-06-06 10:59:58
|
||||
dt64: 2024-06-06 10:59:58.851
|
||||
-- Can be combined with generateSnowflakeID
|
||||
86
tests/queries/0_stateless/01942_snowflakeIDToDateTime.sql
Normal file
86
tests/queries/0_stateless/01942_snowflakeIDToDateTime.sql
Normal file
@ -0,0 +1,86 @@
|
||||
SET session_timezone = 'UTC'; -- disable timezone randomization
|
||||
SET allow_experimental_analyzer = 1; -- The old path formats the result with different whitespaces
|
||||
SET uniform_snowflake_conversion_functions = 1; -- Force-enable uniform snowflake conversion functions (in case this is randomized in CI)
|
||||
|
||||
SELECT '-- Negative tests';
|
||||
SELECT snowflakeIDToDateTime(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT snowflakeIDToDateTime64(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT snowflakeIDToDateTime('invalid_snowflake'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeIDToDateTime64('invalid_snowflake'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeIDToDateTime(123::UInt64, 'invalid_epoch'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeIDToDateTime64(123::UInt64, 'invalid_epoch'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeIDToDateTime(123::UInt64, materialize(42)); -- {serverError ILLEGAL_COLUMN}
|
||||
SELECT snowflakeIDToDateTime64(123::UInt64, materialize(42)); -- {serverError ILLEGAL_COLUMN}
|
||||
SELECT snowflakeIDToDateTime(123::UInt64, 42, 42); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeIDToDateTime64(123::UInt64, 42, 42); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeIDToDateTime(123::UInt64, 42, 'UTC', 'too_many_args'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT snowflakeIDToDateTime64(123::UInt64, 42, 'UTC', 'too_many_args'); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
|
||||
SELECT snowflakeIDToDateTime(123::UInt64) SETTINGS uniform_snowflake_conversion_functions = 0; -- { serverError UNKNOWN_FUNCTION }
|
||||
SELECT snowflakeIDToDateTime64(123::UInt64) SETTINGS uniform_snowflake_conversion_functions = 0; -- { serverError UNKNOWN_FUNCTION }
|
||||
|
||||
SELECT '-- Return type';
|
||||
SELECT toTypeName(snowflakeIDToDateTime(123::UInt64));
|
||||
SELECT toTypeName(snowflakeIDToDateTime64(123::UInt64));
|
||||
|
||||
SELECT '-- Non-const path';
|
||||
-- Two const arguments are mapped to two non-const arguments ('getDefaultImplementationForConstants'), the non-const path is taken
|
||||
|
||||
WITH
|
||||
7204436857747984384 AS sf
|
||||
SELECT
|
||||
sf,
|
||||
snowflakeIDToDateTime(sf) as dt,
|
||||
snowflakeIDToDateTime64(sf) as dt64
|
||||
FORMAT
|
||||
Vertical;
|
||||
|
||||
-- With Twitter Snowflake ID and Twitter epoch
|
||||
WITH
|
||||
1426981498778550272 AS sf,
|
||||
1288834974657 AS epoch
|
||||
SELECT
|
||||
sf,
|
||||
snowflakeIDToDateTime(sf, epoch) as dt,
|
||||
snowflakeIDToDateTime64(sf, epoch) as dt64
|
||||
FORMAT
|
||||
Vertical;
|
||||
|
||||
-- non-default timezone
|
||||
WITH
|
||||
7204436857747984384 AS sf,
|
||||
0 AS epoch, -- default epoch
|
||||
'Asia/Shanghai' AS tz
|
||||
SELECT
|
||||
sf,
|
||||
snowflakeIDToDateTime(sf, epoch, tz) as dt,
|
||||
snowflakeIDToDateTime64(sf, epoch, tz) as dt64
|
||||
FORMAT
|
||||
Vertical;
|
||||
|
||||
SELECT '-- Const path';
|
||||
|
||||
-- The const path can only be tested by const snowflake + const epoch + non-const time-zone. The latter requires a special setting.
|
||||
WITH
|
||||
7204436857747984384 AS sf,
|
||||
0 AS epoch, -- default epoch
|
||||
materialize('Asia/Shanghai') AS tz
|
||||
SELECT
|
||||
sf,
|
||||
snowflakeIDToDateTime(sf, epoch, tz) as dt,
|
||||
snowflakeIDToDateTime64(sf, epoch, tz) as dt64
|
||||
FORMAT
|
||||
Vertical
|
||||
SETTINGS
|
||||
allow_nonconst_timezone_arguments = 1;
|
||||
|
||||
|
||||
SELECT '-- Can be combined with generateSnowflakeID';
|
||||
|
||||
WITH
|
||||
generateSnowflakeID() AS snowflake
|
||||
SELECT
|
||||
snowflakeIDToDateTime(snowflake),
|
||||
snowflakeIDToDateTime64(snowflake)
|
||||
FORMAT
|
||||
Null;
|
||||
@ -1,3 +1,5 @@
|
||||
SET uniform_snowflake_conversion_functions = 0; -- Force-disable uniform snowflake conversion functions (in case this is randomized in CI)
|
||||
|
||||
-- -- Error cases
|
||||
SELECT snowflakeToDateTime(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
SELECT snowflakeToDateTime64(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
|
||||
@ -8,6 +10,9 @@ SELECT snowflakeToDateTime64('abc'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeToDateTime('abc', 123); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
SELECT snowflakeToDateTime64('abc', 123); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
|
||||
|
||||
SELECT snowflakeToDateTime(123::Int64) SETTINGS uniform_snowflake_conversion_functions = 1; -- { serverError DEPRECATED_FUNCTION }
|
||||
SELECT snowflakeToDateTime64(123::Int64) SETTINGS uniform_snowflake_conversion_functions = 1; -- { serverError DEPRECATED_FUNCTION }
|
||||
|
||||
SELECT 'const column';
|
||||
WITH
|
||||
CAST(1426860704886947840 AS Int64) AS i64,
|
||||
|
||||
@ -13,53 +13,84 @@ clickhouse mouse 6
|
||||
-- non-const arguments
|
||||
byteHammingDistance 0
|
||||
byteHammingDistance abc 3
|
||||
byteHammingDistance Jerry 我是谁 9
|
||||
byteHammingDistance abc 3
|
||||
byteHammingDistance abc ab 1
|
||||
byteHammingDistance abc abc 0
|
||||
byteHammingDistance abc bc 3
|
||||
byteHammingDistance clickhouse mouse 10
|
||||
byteHammingDistance 我是谁 Tom 9
|
||||
byteHammingDistance 我是谁 我是我 3
|
||||
editDistance 0
|
||||
editDistance abc 3
|
||||
editDistance Jerry 我是谁 9
|
||||
editDistance abc 3
|
||||
editDistance abc ab 1
|
||||
editDistance abc abc 0
|
||||
editDistance abc bc 1
|
||||
editDistance clickhouse mouse 6
|
||||
editDistance 我是谁 Tom 9
|
||||
editDistance 我是谁 我是我 3
|
||||
editDistanceUTF8 0
|
||||
editDistanceUTF8 abc 3
|
||||
editDistanceUTF8 Jerry 我是谁 5
|
||||
editDistanceUTF8 abc 3
|
||||
editDistanceUTF8 abc ab 1
|
||||
editDistanceUTF8 abc abc 0
|
||||
editDistanceUTF8 abc bc 1
|
||||
editDistanceUTF8 clickhouse mouse 6
|
||||
editDistanceUTF8 我是谁 Tom 3
|
||||
editDistanceUTF8 我是谁 我是我 1
|
||||
damerauLevenshteinDistance 0
|
||||
damerauLevenshteinDistance abc 3
|
||||
damerauLevenshteinDistance Jerry 我是谁 9
|
||||
damerauLevenshteinDistance abc 3
|
||||
damerauLevenshteinDistance abc ab 1
|
||||
damerauLevenshteinDistance abc abc 0
|
||||
damerauLevenshteinDistance abc bc 1
|
||||
damerauLevenshteinDistance clickhouse mouse 6
|
||||
damerauLevenshteinDistance 我是谁 Tom 9
|
||||
damerauLevenshteinDistance 我是谁 我是我 3
|
||||
stringJaccardIndex 0
|
||||
stringJaccardIndex abc 0
|
||||
stringJaccardIndex Jerry 我是谁 0
|
||||
stringJaccardIndex abc 0
|
||||
stringJaccardIndex abc ab 0.6666666666666666
|
||||
stringJaccardIndex abc abc 1
|
||||
stringJaccardIndex abc bc 0.6666666666666666
|
||||
stringJaccardIndex clickhouse mouse 0.4
|
||||
stringJaccardIndex 我是谁 Tom 0
|
||||
stringJaccardIndex 我是谁 我是我 0.625
|
||||
stringJaccardIndexUTF8 0
|
||||
stringJaccardIndexUTF8 abc 0
|
||||
stringJaccardIndexUTF8 Jerry 我是谁 0
|
||||
stringJaccardIndexUTF8 abc 0
|
||||
stringJaccardIndexUTF8 abc ab 0.6666666666666666
|
||||
stringJaccardIndexUTF8 abc abc 1
|
||||
stringJaccardIndexUTF8 abc bc 0.6666666666666666
|
||||
stringJaccardIndexUTF8 clickhouse mouse 0.4
|
||||
stringJaccardIndexUTF8 我是谁 Tom 0
|
||||
stringJaccardIndexUTF8 我是谁 我是我 0.6666666666666666
|
||||
jaroSimilarity 0
|
||||
jaroSimilarity abc 3
|
||||
jaroSimilarity Jerry 我是谁 0
|
||||
jaroSimilarity abc 3
|
||||
jaroSimilarity abc ab 0.8888888888888888
|
||||
jaroSimilarity abc abc 1
|
||||
jaroSimilarity abc bc 0
|
||||
jaroSimilarity clickhouse mouse 0
|
||||
jaroSimilarity 我是谁 Tom 0
|
||||
jaroSimilarity 我是谁 我是我 0.7777777777777777
|
||||
jaroWinklerSimilarity 0
|
||||
jaroWinklerSimilarity abc 3
|
||||
jaroWinklerSimilarity Jerry 我是谁 0
|
||||
jaroWinklerSimilarity abc 3
|
||||
jaroWinklerSimilarity abc ab 0.9111111111111111
|
||||
jaroWinklerSimilarity abc abc 1
|
||||
jaroWinklerSimilarity abc bc 0
|
||||
jaroWinklerSimilarity clickhouse mouse 0
|
||||
jaroWinklerSimilarity 我是谁 Tom 0
|
||||
jaroWinklerSimilarity 我是谁 我是我 0.8666666666666666
|
||||
-- Special UTF-8 tests
|
||||
0.4
|
||||
0
|
||||
|
||||
@ -26,11 +26,12 @@ CREATE TABLE t
|
||||
) ENGINE = MergeTree ORDER BY s1;
|
||||
|
||||
-- actual test cases
|
||||
INSERT INTO t VALUES ('', '') ('abc', '') ('', 'abc') ('abc', 'abc') ('abc', 'ab') ('abc', 'bc') ('clickhouse', 'mouse');
|
||||
INSERT INTO t VALUES ('', '') ('abc', '') ('', 'abc') ('abc', 'abc') ('abc', 'ab') ('abc', 'bc') ('clickhouse', 'mouse') ('我是谁', 'Tom') ('Jerry', '我是谁') ('我是谁', '我是我');
|
||||
|
||||
SELECT '-- non-const arguments';
|
||||
SELECT 'byteHammingDistance', s1, s2, byteHammingDistance(s1, s2) FROM t ORDER BY ALL;
|
||||
SELECT 'editDistance', s1, s2, editDistance(s1, s2) FROM t ORDER BY ALL;
|
||||
SELECT 'editDistanceUTF8', s1, s2, editDistanceUTF8(s1, s2) FROM t ORDER BY ALL;
|
||||
SELECT 'damerauLevenshteinDistance', s1, s2, damerauLevenshteinDistance(s1, s2) FROM t ORDER BY ALL;
|
||||
SELECT 'stringJaccardIndex', s1, s2, stringJaccardIndex(s1, s2) FROM t ORDER BY ALL;
|
||||
SELECT 'stringJaccardIndexUTF8', s1, s2, stringJaccardIndexUTF8(s1, s2) FROM t ORDER BY ALL;
|
||||
|
||||
@ -0,0 +1,5 @@
|
||||
1
|
||||
3168 8613
|
||||
[] ['SELECT(a, b) ON default.d_03168_query_log']
|
||||
[] []
|
||||
['SELECT(a, b) ON default.d_03168_query_log'] []
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user