mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-29 02:52:13 +00:00
Merge branch 'master' into nv/parts-uuid-move-shard
This commit is contained in:
alesapin
GitHub
commit
4d6c71801c
@ -8,7 +8,7 @@ ClickHouse® is an open-source column-oriented database management system that a
|
||||
* [Tutorial](https://clickhouse.tech/docs/en/getting_started/tutorial/) shows how to set up and query small ClickHouse cluster.
|
||||
* [Documentation](https://clickhouse.tech/docs/en/) provides more in-depth information.
|
||||
* [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
|
||||
* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-nwwakmk4-xOJ6cdy0sJC3It8j348~IA) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time.
|
||||
* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-qfort0u8-TWqK4wIP0YSdoDE0btKa1w) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time.
|
||||
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events.
|
||||
* [Code Browser](https://clickhouse.tech/codebrowser/html_report/ClickHouse/index.html) with syntax highlight and navigation.
|
||||
* [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
|
||||
|
||||
2
contrib/grpc
vendored
2
contrib/grpc
vendored
@ -1 +1 @@
|
||||
Subproject commit 5b79aae85c515e0df4abfb7b1e07975fdc7cecc1
|
||||
Subproject commit 60c986e15cae70aade721d26badabab1f822fdd6
|
||||
@ -17,6 +17,7 @@ To define LDAP server you must add `ldap_servers` section to the `config.xml`.
|
||||
<yandex>
|
||||
<!- ... -->
|
||||
<ldap_servers>
|
||||
<!- Typical LDAP server. -->
|
||||
<my_ldap_server>
|
||||
<host>localhost</host>
|
||||
<port>636</port>
|
||||
@ -31,6 +32,18 @@ To define LDAP server you must add `ldap_servers` section to the `config.xml`.
|
||||
<tls_ca_cert_dir>/path/to/tls_ca_cert_dir</tls_ca_cert_dir>
|
||||
<tls_cipher_suite>ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384</tls_cipher_suite>
|
||||
</my_ldap_server>
|
||||
|
||||
<!- Typical Active Directory with configured user DN detection for further role mapping. -->
|
||||
<my_ad_server>
|
||||
<host>localhost</host>
|
||||
<port>389</port>
|
||||
<bind_dn>EXAMPLE\{user_name}</bind_dn>
|
||||
<user_dn_detection>
|
||||
<base_dn>CN=Users,DC=example,DC=com</base_dn>
|
||||
<search_filter>(&(objectClass=user)(sAMAccountName={user_name}))</search_filter>
|
||||
</user_dn_detection>
|
||||
<enable_tls>no</enable_tls>
|
||||
</my_ad_server>
|
||||
</ldap_servers>
|
||||
</yandex>
|
||||
```
|
||||
@ -43,6 +56,15 @@ Note, that you can define multiple LDAP servers inside the `ldap_servers` sectio
|
||||
- `port` — LDAP server port, default is `636` if `enable_tls` is set to `true`, `389` otherwise.
|
||||
- `bind_dn` — Template used to construct the DN to bind to.
|
||||
- The resulting DN will be constructed by replacing all `{user_name}` substrings of the template with the actual user name during each authentication attempt.
|
||||
- `user_dn_detection` - Section with LDAP search parameters for detecting the actual user DN of the bound user.
|
||||
- This is mainly used in search filters for further role mapping when the server is Active Directory. The resulting user DN will be used when replacing `{user_dn}` substrings wherever they are allowed. By default, user DN is set equal to bind DN, but once search is performed, it will be updated with to the actual detected user DN value.
|
||||
- `base_dn` - Template used to construct the base DN for the LDAP search.
|
||||
- The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` substrings of the template with the actual user name and bind DN during the LDAP search.
|
||||
- `scope` - Scope of the LDAP search.
|
||||
- Accepted values are: `base`, `one_level`, `children`, `subtree` (the default).
|
||||
- `search_filter` - Template used to construct the search filter for the LDAP search.
|
||||
- The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings of the template with the actual user name, bind DN, and base DN during the LDAP search.
|
||||
- Note, that the special characters must be escaped properly in XML.
|
||||
- `verification_cooldown` — A period of time, in seconds, after a successful bind attempt, during which the user will be assumed to be successfully authenticated for all consecutive requests without contacting the LDAP server.
|
||||
- Specify `0` (the default) to disable caching and force contacting the LDAP server for each authentication request.
|
||||
- `enable_tls` — A flag to trigger the use of the secure connection to the LDAP server.
|
||||
@ -107,7 +129,7 @@ Goes into `config.xml`.
|
||||
<yandex>
|
||||
<!- ... -->
|
||||
<user_directories>
|
||||
<!- ... -->
|
||||
<!- Typical LDAP server. -->
|
||||
<ldap>
|
||||
<server>my_ldap_server</server>
|
||||
<roles>
|
||||
@ -122,6 +144,18 @@ Goes into `config.xml`.
|
||||
<prefix>clickhouse_</prefix>
|
||||
</role_mapping>
|
||||
</ldap>
|
||||
|
||||
<!- Typical Active Directory with role mapping that relies on the detected user DN. -->
|
||||
<ldap>
|
||||
<server>my_ad_server</server>
|
||||
<role_mapping>
|
||||
<base_dn>CN=Users,DC=example,DC=com</base_dn>
|
||||
<attribute>CN</attribute>
|
||||
<scope>subtree</scope>
|
||||
<search_filter>(&(objectClass=group)(member={user_dn}))</search_filter>
|
||||
<prefix>clickhouse_</prefix>
|
||||
</role_mapping>
|
||||
</ldap>
|
||||
</user_directories>
|
||||
</yandex>
|
||||
```
|
||||
@ -137,13 +171,13 @@ Note that `my_ldap_server` referred in the `ldap` section inside the `user_direc
|
||||
- When a user authenticates, while still bound to LDAP, an LDAP search is performed using `search_filter` and the name of the logged-in user. For each entry found during that search, the value of the specified attribute is extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement.
|
||||
- There can be multiple `role_mapping` sections defined inside the same `ldap` section. All of them will be applied.
|
||||
- `base_dn` — Template used to construct the base DN for the LDAP search.
|
||||
- The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` substrings of the template with the actual user name and bind DN during each LDAP search.
|
||||
- The resulting DN will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{user_dn}` substrings of the template with the actual user name, bind DN, and user DN during each LDAP search.
|
||||
- `scope` — Scope of the LDAP search.
|
||||
- Accepted values are: `base`, `one_level`, `children`, `subtree` (the default).
|
||||
- `search_filter` — Template used to construct the search filter for the LDAP search.
|
||||
- The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}` and `{base_dn}` substrings of the template with the actual user name, bind DN and base DN during each LDAP search.
|
||||
- The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, `{user_dn}`, and `{base_dn}` substrings of the template with the actual user name, bind DN, user DN, and base DN during each LDAP search.
|
||||
- Note, that the special characters must be escaped properly in XML.
|
||||
- `attribute` — Attribute name whose values will be returned by the LDAP search.
|
||||
- `attribute` — Attribute name whose values will be returned by the LDAP search. `cn`, by default.
|
||||
- `prefix` — Prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. The prefix will be removed from the original strings and the resulting strings will be treated as local role names. Empty by default.
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap/) <!--hide-->
|
||||
|
||||
@ -253,7 +253,7 @@ windowFunnel(window, [mode, [mode, ... ]])(timestamp, cond1, cond2, ..., condN)
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `window` — Length of the sliding window, it is the time interval between first condition and last condition. The unit of `window` depends on the `timestamp` itself and varies. Determined using the expression `timestamp of cond1 <= timestamp of cond2 <= ... <= timestamp of condN <= timestamp of cond1 + window`.
|
||||
- `window` — Length of the sliding window, it is the time interval between the first and the last condition. The unit of `window` depends on the `timestamp` itself and varies. Determined using the expression `timestamp of cond1 <= timestamp of cond2 <= ... <= timestamp of condN <= timestamp of cond1 + window`.
|
||||
- `mode` — It is an optional argument. One or more modes can be set.
|
||||
- `'strict'` — If same condition holds for sequence of events then such non-unique events would be skipped.
|
||||
- `'strict_order'` — Don't allow interventions of other events. E.g. in the case of `A->B->D->C`, it stops finding `A->B->C` at the `D` and the max event level is 2.
|
||||
@ -312,7 +312,7 @@ FROM
|
||||
GROUP BY user_id
|
||||
)
|
||||
GROUP BY level
|
||||
ORDER BY level ASC
|
||||
ORDER BY level ASC;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
@ -422,7 +422,7 @@ Type: [UInt8](../../sql-reference/data-types/int-uint.md).
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT isIPAddressInRange('127.0.0.1', '127.0.0.0/8')
|
||||
SELECT isIPAddressInRange('127.0.0.1', '127.0.0.0/8');
|
||||
```
|
||||
|
||||
Result:
|
||||
@ -436,7 +436,7 @@ Result:
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT isIPAddressInRange('127.0.0.1', 'ffff::/16')
|
||||
SELECT isIPAddressInRange('127.0.0.1', 'ffff::/16');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
@ -373,7 +373,7 @@ This function accepts a number or date or date with time, and returns a FixedStr
|
||||
|
||||
## reinterpretAsUUID {#reinterpretasuuid}
|
||||
|
||||
This function accepts 16 bytes string, and returns UUID containing bytes representing the corresponding value in network byte order (big-endian). If the string isn't long enough, the functions work as if the string is padded with the necessary number of null bytes to the end. If the string longer than 16 bytes, the extra bytes at the end are ignored.
|
||||
Accepts 16 bytes string and returns UUID containing bytes representing the corresponding value in network byte order (big-endian). If the string isn't long enough, the function works as if the string is padded with the necessary number of null bytes to the end. If the string longer than 16 bytes, the extra bytes at the end are ignored.
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -429,7 +429,24 @@ Result:
|
||||
|
||||
## reinterpret(x, T) {#type_conversion_function-reinterpret}
|
||||
|
||||
Use the same source in-memory bytes sequence for `x` value and reinterpret it to destination type
|
||||
Uses the same source in-memory bytes sequence for `x` value and reinterprets it to destination type.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
reinterpret(x, type)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` — Any type.
|
||||
- `type` — Destination type. [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Destination type value.
|
||||
|
||||
**Examples**
|
||||
|
||||
Query:
|
||||
```sql
|
||||
@ -448,11 +465,27 @@ Result:
|
||||
|
||||
## CAST(x, T) {#type_conversion_function-cast}
|
||||
|
||||
Converts input value `x` to the `T` data type. Unlike to `reinterpret` function use external representation of `x` value.
|
||||
Converts input value `x` to the `T` data type. Unlike to `reinterpret` function, type conversion is performed in a natural way.
|
||||
|
||||
The syntax `CAST(x AS t)` is also supported.
|
||||
|
||||
Note, that if value `x` does not fit the bounds of type T, the function overflows. For example, CAST(-1, 'UInt8') returns 255.
|
||||
!!! note "Note"
|
||||
If value `x` does not fit the bounds of type `T`, the function overflows. For example, `CAST(-1, 'UInt8')` returns `255`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
CAST(x, T)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` — Any type.
|
||||
- `T` — Destination type. [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Destination type value.
|
||||
|
||||
**Examples**
|
||||
|
||||
@ -460,9 +493,9 @@ Query:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
cast(toInt8(-1), 'UInt8') AS cast_int_to_uint,
|
||||
cast(toInt8(1), 'Float32') AS cast_int_to_float,
|
||||
cast('1', 'UInt32') AS cast_string_to_int
|
||||
CAST(toInt8(-1), 'UInt8') AS cast_int_to_uint,
|
||||
CAST(toInt8(1), 'Float32') AS cast_int_to_float,
|
||||
CAST('1', 'UInt32') AS cast_string_to_int;
|
||||
```
|
||||
|
||||
Result:
|
||||
@ -492,7 +525,7 @@ Result:
|
||||
└─────────────────────┴─────────────────────┴────────────┴─────────────────────┴───────────────────────────┘
|
||||
```
|
||||
|
||||
Conversion to FixedString(N) only works for arguments of type String or FixedString(N).
|
||||
Conversion to FixedString(N) only works for arguments of type [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md).
|
||||
|
||||
Type conversion to [Nullable](../../sql-reference/data-types/nullable.md) and back is supported.
|
||||
|
||||
@ -1038,7 +1071,7 @@ Result:
|
||||
|
||||
## parseDateTime64BestEffort {#parsedatetime64besteffort}
|
||||
|
||||
Same as [parseDateTimeBestEffort](#parsedatetimebesteffort) function but also parse milliseconds and microseconds and return `DateTime64(3)` or `DateTime64(6)` data types.
|
||||
Same as [parseDateTimeBestEffort](#parsedatetimebesteffort) function but also parse milliseconds and microseconds and returns [DateTime](../../sql-reference/functions/type-conversion-functions.md#data_type-datetime) data type.
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -1049,9 +1082,13 @@ parseDateTime64BestEffort(time_string [, precision [, time_zone]])
|
||||
**Parameters**
|
||||
|
||||
- `time_string` — String containing a date or date with time to convert. [String](../../sql-reference/data-types/string.md).
|
||||
- `precision` — `3` for milliseconds, `6` for microseconds. Default `3`. Optional [UInt8](../../sql-reference/data-types/int-uint.md).
|
||||
- `precision` — Required precision. `3` — for milliseconds, `6` — for microseconds. Default — `3`. Optional. [UInt8](../../sql-reference/data-types/int-uint.md).
|
||||
- `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- `time_string` converted to the [DateTime](../../sql-reference/data-types/datetime.md) data type.
|
||||
|
||||
**Examples**
|
||||
|
||||
Query:
|
||||
@ -1064,7 +1101,7 @@ UNION ALL
|
||||
SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',6) AS a, toTypeName(a) AS t
|
||||
UNION ALL
|
||||
SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',3,'Europe/Moscow') AS a, toTypeName(a) AS t
|
||||
FORMAT PrettyCompactMonoBlcok
|
||||
FORMAT PrettyCompactMonoBlock;
|
||||
```
|
||||
|
||||
Result:
|
||||
@ -1131,12 +1168,14 @@ Result:
|
||||
|
||||
## toUnixTimestamp64Nano {#tounixtimestamp64nano}
|
||||
|
||||
Converts a `DateTime64` to a `Int64` value with fixed sub-second precision.
|
||||
Input value is scaled up or down appropriately depending on it precision. Please note that output value is a timestamp in UTC, not in timezone of `DateTime64`.
|
||||
Converts a `DateTime64` to a `Int64` value with fixed sub-second precision. Input value is scaled up or down appropriately depending on it precision.
|
||||
|
||||
!!! info "Note"
|
||||
The output value is a timestamp in UTC, not in the timezone of `DateTime64`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
```sql
|
||||
toUnixTimestamp64Milli(value)
|
||||
```
|
||||
|
||||
@ -1152,7 +1191,7 @@ toUnixTimestamp64Milli(value)
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
```sql
|
||||
WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64
|
||||
SELECT toUnixTimestamp64Milli(dt64);
|
||||
```
|
||||
@ -1298,4 +1337,3 @@ Result:
|
||||
│ 2,"good" │
|
||||
└───────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
|
||||
@ -316,7 +316,7 @@ Allows executing [CREATE](../../sql-reference/statements/create/index.md) and [A
|
||||
|
||||
Allows executing [DROP](../../sql-reference/statements/misc.md#drop) and [DETACH](../../sql-reference/statements/misc.md#detach) queries according to the following hierarchy of privileges:
|
||||
|
||||
- `DROP`. Level:
|
||||
- `DROP`. Level: `GROUP`
|
||||
- `DROP DATABASE`. Level: `DATABASE`
|
||||
- `DROP TABLE`. Level: `TABLE`
|
||||
- `DROP VIEW`. Level: `VIEW`
|
||||
|
||||
@ -253,7 +253,7 @@ windowFunnel(window, [mode, [mode, ... ]])(timestamp, cond1, cond2, ..., condN)
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `window` — ширина скользящего окна по времени. Единица измерения зависит от `timestamp` и может варьироваться. Должно соблюдаться условие `timestamp события cond2 <= timestamp события cond1 + window`.
|
||||
- `window` — ширина скользящего окна по времени. Это время между первым и последним условием. Единица измерения зависит от `timestamp` и может варьироваться. Должно соблюдаться условие `timestamp события cond1 <= timestamp события cond2 <= ... <= timestamp события condN <= timestamp события cond1 + window`.
|
||||
- `mode` — необязательный параметр. Может быть установленно несколько значений одновременно.
|
||||
- `'strict'` — не учитывать подряд идущие повторяющиеся события.
|
||||
- `'strict_order'` — запрещает посторонние события в искомой последовательности. Например, при поиске цепочки `A->B->C` в `A->B->D->C` поиск будет остановлен на `D` и функция вернет 2.
|
||||
@ -311,7 +311,7 @@ FROM
|
||||
GROUP BY user_id
|
||||
)
|
||||
GROUP BY level
|
||||
ORDER BY level ASC
|
||||
ORDER BY level ASC;
|
||||
```
|
||||
|
||||
## retention {#retention}
|
||||
|
||||
@ -397,9 +397,9 @@ SELECT addr, isIPv6String(addr) FROM ( SELECT ['::', '1111::ffff', '::ffff:127.0
|
||||
|
||||
## isIPAddressInRange {#isipaddressinrange}
|
||||
|
||||
Проверяет попадает ли IP адрес в интервал, заданный в [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) нотации.
|
||||
Проверяет, попадает ли IP адрес в интервал, заданный в нотации [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing).
|
||||
|
||||
**Syntax**
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
isIPAddressInRange(address, prefix)
|
||||
@ -409,7 +409,7 @@ isIPAddressInRange(address, prefix)
|
||||
**Аргументы**
|
||||
|
||||
- `address` — IPv4 или IPv6 адрес. [String](../../sql-reference/data-types/string.md).
|
||||
- `prefix` — IPv4 или IPv6 подсеть, заданная в CIDR нотации. [String](../../sql-reference/data-types/string.md).
|
||||
- `prefix` — IPv4 или IPv6 подсеть, заданная в нотации CIDR. [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
@ -422,7 +422,7 @@ isIPAddressInRange(address, prefix)
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT isIPAddressInRange('127.0.0.1', '127.0.0.0/8')
|
||||
SELECT isIPAddressInRange('127.0.0.1', '127.0.0.0/8');
|
||||
```
|
||||
|
||||
Результат:
|
||||
@ -436,7 +436,7 @@ SELECT isIPAddressInRange('127.0.0.1', '127.0.0.0/8')
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT isIPAddressInRange('127.0.0.1', 'ffff::/16')
|
||||
SELECT isIPAddressInRange('127.0.0.1', 'ffff::/16');
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
@ -369,7 +369,7 @@ SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut;
|
||||
|
||||
## reinterpretAsUUID {#reinterpretasuuid}
|
||||
|
||||
Функция принимает шестнадцатибайтную строку и интерпретирует ее байты в network order (big-endian). Если строка имеет недостаточную длину, то функция работает так, как будто строка дополнена необходимым количетсвом нулевых байт с конца. Если строка длиннее, чем шестнадцать байт, то игнорируются лишние байты с конца.
|
||||
Функция принимает строку из 16 байт и интерпретирует ее байты в порядок от старшего к младшему. Если строка имеет недостаточную длину, то функция работает так, как будто строка дополнена необходимым количеством нулевых байтов с конца. Если строка длиннее, чем 16 байтов, то лишние байты с конца игнорируются.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
@ -425,9 +425,27 @@ SELECT uuid = uuid2;
|
||||
|
||||
## reinterpret(x, T) {#type_conversion_function-reinterpret}
|
||||
|
||||
Использует туже самую исходную последовательность байт в памяти для значения `x` и переинтерпретирует ее как конечный тип данных
|
||||
Использует ту же самую исходную последовательность байтов в памяти для значения `x` и интерпретирует ее как конечный тип данных `T`.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
reinterpret(x, type)
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `x` — любой тип данных.
|
||||
- `type` — конечный тип данных. [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
- Значение конечного типа данных.
|
||||
|
||||
**Примеры**
|
||||
|
||||
Запрос:
|
||||
|
||||
```sql
|
||||
SELECT reinterpret(toInt8(-1), 'UInt8') as int_to_uint,
|
||||
reinterpret(toInt8(1), 'Float32') as int_to_float,
|
||||
@ -448,7 +466,23 @@ SELECT reinterpret(toInt8(-1), 'UInt8') as int_to_uint,
|
||||
|
||||
Поддерживается также синтаксис `CAST(x AS t)`.
|
||||
|
||||
Обратите внимание, что если значение `x` не может быть преобразовано к типу `T`, возникает переполнение. Например, `CAST(-1, 'UInt8')` возвращает 255.
|
||||
!!! warning "Предупреждение"
|
||||
Если значение `x` не может быть преобразовано к типу `T`, возникает переполнение. Например, `CAST(-1, 'UInt8')` возвращает 255.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
CAST(x, T)
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `x` — любой тип данных.
|
||||
- `T` — конечный тип данных. [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
- Значение конечного типа данных.
|
||||
|
||||
**Примеры**
|
||||
|
||||
@ -456,9 +490,9 @@ SELECT reinterpret(toInt8(-1), 'UInt8') as int_to_uint,
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
cast(toInt8(-1), 'UInt8') AS cast_int_to_uint,
|
||||
cast(toInt8(1), 'Float32') AS cast_int_to_float,
|
||||
cast('1', 'UInt32') AS cast_string_to_int
|
||||
CAST(toInt8(-1), 'UInt8') AS cast_int_to_uint,
|
||||
CAST(toInt8(1), 'Float32') AS cast_int_to_float,
|
||||
CAST('1', 'UInt32') AS cast_string_to_int
|
||||
```
|
||||
|
||||
Результат:
|
||||
@ -488,9 +522,9 @@ SELECT
|
||||
└─────────────────────┴─────────────────────┴────────────┴─────────────────────┴───────────────────────────┘
|
||||
```
|
||||
|
||||
Преобразование в FixedString(N) работает только для аргументов типа String или FixedString(N).
|
||||
Преобразование в FixedString(N) работает только для аргументов типа [String](../../sql-reference/data-types/string.md) или [FixedString](../../sql-reference/data-types/fixedstring.md).
|
||||
|
||||
Поддержано преобразование к типу [Nullable](../../sql-reference/functions/type-conversion-functions.md) и обратно.
|
||||
Поддерживается преобразование к типу [Nullable](../../sql-reference/functions/type-conversion-functions.md) и обратно.
|
||||
|
||||
**Примеры**
|
||||
|
||||
@ -860,7 +894,7 @@ AS parseDateTimeBestEffortUS;
|
||||
## parseDateTimeBestEffortOrZero {#parsedatetimebesteffortorzero}
|
||||
## parseDateTime32BestEffortOrZero {#parsedatetime32besteffortorzero}
|
||||
|
||||
Работает также как [parseDateTimeBestEffort](#parsedatetimebesteffort), но возвращает нулевую дату или нулевую дату и время когда получает формат даты который не может быть обработан.
|
||||
Работает аналогично функции [parseDateTimeBestEffort](#parsedatetimebesteffort), но возвращает нулевое значение, если формат даты не может быть обработан.
|
||||
|
||||
## parseDateTimeBestEffortUSOrNull {#parsedatetimebesteffortusornull}
|
||||
|
||||
@ -1036,19 +1070,23 @@ SELECT parseDateTimeBestEffortUSOrZero('02.2021') AS parseDateTimeBestEffortUSOr
|
||||
|
||||
## parseDateTime64BestEffort {#parsedatetime64besteffort}
|
||||
|
||||
Работает также как функция [parseDateTimeBestEffort](#parsedatetimebesteffort) но также понимамет милисекунды и микросекунды и возвращает `DateTime64(3)` или `DateTime64(6)` типы данных в зависимости от заданной точности.
|
||||
Работает аналогично функции [parseDateTimeBestEffort](#parsedatetimebesteffort), но также принимает миллисекунды и микросекунды. Возвращает тип данных [DateTime](../../sql-reference/functions/type-conversion-functions.md#data_type-datetime).
|
||||
|
||||
**Syntax**
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
parseDateTime64BestEffort(time_string [, precision [, time_zone]])
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
**Аргументы**
|
||||
|
||||
- `time_string` — String containing a date or date with time to convert. [String](../../sql-reference/data-types/string.md).
|
||||
- `precision` — `3` for milliseconds, `6` for microseconds. Default `3`. Optional [UInt8](../../sql-reference/data-types/int-uint.md).
|
||||
- `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md).
|
||||
- `time_string` — строка, содержащая дату или дату со временем, которые нужно преобразовать. [String](../../sql-reference/data-types/string.md).
|
||||
- `precision` — требуемая точность: `3` — для миллисекунд, `6` — для микросекунд. По умолчанию — `3`. Необязательный. [UInt8](../../sql-reference/data-types/int-uint.md).
|
||||
- `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). Разбирает значение `time_string` в зависимости от часового пояса. Необязательный. [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
- `time_string`, преобразованная в тип данных [DateTime](../../sql-reference/data-types/datetime.md).
|
||||
|
||||
**Примеры**
|
||||
|
||||
@ -1062,7 +1100,7 @@ UNION ALL
|
||||
SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',6) AS a, toTypeName(a) AS t
|
||||
UNION ALL
|
||||
SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',3,'Europe/Moscow') AS a, toTypeName(a) AS t
|
||||
FORMAT PrettyCompactMonoBlcok
|
||||
FORMAT PrettyCompactMonoBlock;
|
||||
```
|
||||
|
||||
Результат:
|
||||
@ -1078,12 +1116,11 @@ FORMAT PrettyCompactMonoBlcok
|
||||
|
||||
## parseDateTime64BestEffortOrNull {#parsedatetime32besteffortornull}
|
||||
|
||||
Работает также как функция [parseDateTime64BestEffort](#parsedatetime64besteffort) но возвращает `NULL` когда встречает формат даты который не может обработать.
|
||||
Работает аналогично функции [parseDateTime64BestEffort](#parsedatetime64besteffort), но возвращает `NULL`, если формат даты не может быть обработан.
|
||||
|
||||
## parseDateTime64BestEffortOrZero {#parsedatetime64besteffortorzero}
|
||||
|
||||
Работает также как функция [parseDateTime64BestEffort](#parsedatetimebesteffort) но возвращает "нулевую" дату и время когда встречает формат даты который не может обработать.
|
||||
|
||||
Работает аналогично функции [parseDateTime64BestEffort](#parsedatetimebesteffort), но возвращает нулевую дату и время, если формат даты не может быть обработан.
|
||||
|
||||
## toLowCardinality {#tolowcardinality}
|
||||
|
||||
@ -1130,11 +1167,14 @@ SELECT toLowCardinality('1');
|
||||
## toUnixTimestamp64Nano {#tounixtimestamp64nano}
|
||||
|
||||
Преобразует значение `DateTime64` в значение `Int64` с фиксированной точностью менее одной секунды.
|
||||
Входное значение округляется соответствующим образом вверх или вниз в зависимости от его точности. Обратите внимание, что возвращаемое значение - это временная метка в UTC, а не в часовом поясе `DateTime64`.
|
||||
Входное значение округляется соответствующим образом вверх или вниз в зависимости от его точности.
|
||||
|
||||
!!! info "Примечание"
|
||||
Возвращаемое значение — это временная метка в UTC, а не в часовом поясе `DateTime64`.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
```sql
|
||||
toUnixTimestamp64Milli(value)
|
||||
```
|
||||
|
||||
@ -1150,7 +1190,7 @@ toUnixTimestamp64Milli(value)
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
```sql
|
||||
WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64
|
||||
SELECT toUnixTimestamp64Milli(dt64);
|
||||
```
|
||||
@ -1296,4 +1336,3 @@ FROM numbers(3);
|
||||
│ 2,"good" │
|
||||
└───────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
|
||||
@ -51,5 +51,5 @@ The easiest way to see the result is to use `--livereload=8888` argument of buil
|
||||
|
||||
At the moment there’s no easy way to do just that, but you can consider:
|
||||
|
||||
- To hit the “Watch” button on top of GitHub web interface to know as early as possible, even during pull request. Alternative to this is `#github-activity` channel of [public ClickHouse Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-nwwakmk4-xOJ6cdy0sJC3It8j348~IA).
|
||||
- To hit the “Watch” button on top of GitHub web interface to know as early as possible, even during pull request. Alternative to this is `#github-activity` channel of [public ClickHouse Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-qfort0u8-TWqK4wIP0YSdoDE0btKa1w).
|
||||
- Some search engines allow to subscribe on specific website changes via email and you can opt-in for that for https://clickhouse.tech.
|
||||
|
||||
@ -155,10 +155,6 @@ def build_website(args):
|
||||
os.path.join(args.src_dir, 'utils', 'list-versions', 'version_date.tsv'),
|
||||
os.path.join(args.output_dir, 'data', 'version_date.tsv'))
|
||||
|
||||
shutil.copy2(
|
||||
os.path.join(args.website_dir, 'js', 'embedd.min.js'),
|
||||
os.path.join(args.output_dir, 'js', 'embedd.min.js'))
|
||||
|
||||
for root, _, filenames in os.walk(args.output_dir):
|
||||
for filename in filenames:
|
||||
if filename == 'main.html':
|
||||
|
||||
@ -7,11 +7,11 @@ toc_title: ODBC
|
||||
|
||||
# ODBC {#table-engine-odbc}
|
||||
|
||||
允许ClickHouse通过以下方式连接到外部数据库 [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity).
|
||||
允许ClickHouse通过[ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity)方式连接到外部数据库.
|
||||
|
||||
为了安全地实现ODBC连接，ClickHouse使用单独的程序 `clickhouse-odbc-bridge`. 如果直接从ODBC驱动程序加载 `clickhouse-server`，驱动程序问题可能会导致ClickHouse服务器崩溃。 ClickHouse自动启动 `clickhouse-odbc-bridge` 当它是必需的。 ODBC桥程序是从相同的软件包作为安装 `clickhouse-server`.
|
||||
为了安全地实现ODBC连接，ClickHouse使用了一个独立程序 `clickhouse-odbc-bridge`. 如果ODBC驱动程序是直接从 `clickhouse-server`中加载的，那么驱动问题可能会导致ClickHouse服务崩溃。 当有需要时，ClickHouse会自动启动 `clickhouse-odbc-bridge`。 ODBC桥梁程序与`clickhouse-server`来自相同的安装包.
|
||||
|
||||
该引擎支持 [可为空](../../../sql-reference/data-types/nullable.md) 数据类型。
|
||||
该引擎支持 [可为空](../../../sql-reference/data-types/nullable.md) 的数据类型。
|
||||
|
||||
## 创建表 {#creating-a-table}
|
||||
|
||||
@ -25,14 +25,14 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
ENGINE = ODBC(connection_settings, external_database, external_table)
|
||||
```
|
||||
|
||||
请参阅的详细说明 [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) 查询。
|
||||
详情请见 [CREATE TABLE](../../../sql-reference/statements/create.md#create-table-query) 查询。
|
||||
|
||||
表结构可以与源表结构不同:
|
||||
|
||||
- 列名应与源表中的列名相同，但您可以按任何顺序使用其中的一些列。
|
||||
- 列类型可能与源表中的列类型不同。 ClickHouse尝试 [投](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) ClickHouse数据类型的值。
|
||||
- 列类型可能与源表中的列类型不同。 ClickHouse尝试将数值[映射](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) 到ClickHouse的数据类型。
|
||||
|
||||
**发动机参数**
|
||||
**引擎参数**
|
||||
|
||||
- `connection_settings` — Name of the section with connection settings in the `odbc.ini` 文件
|
||||
- `external_database` — Name of a database in an external DBMS.
|
||||
@ -40,13 +40,13 @@ ENGINE = ODBC(connection_settings, external_database, external_table)
|
||||
|
||||
## 用法示例 {#usage-example}
|
||||
|
||||
**通过ODBC从本地MySQL安装中检索数据**
|
||||
**通过ODBC从本地安装的MySQL中检索数据**
|
||||
|
||||
此示例检查Ubuntu Linux18.04和MySQL服务器5.7。
|
||||
本示例针对Ubuntu Linux18.04和MySQL服务器5.7进行检查。
|
||||
|
||||
确保安装了unixODBC和MySQL连接器。
|
||||
请确保安装了unixODBC和MySQL连接器。
|
||||
|
||||
默认情况下（如果从软件包安装），ClickHouse以用户身份启动 `clickhouse`. 因此，您需要在MySQL服务器中创建和配置此用户。
|
||||
默认情况下（如果从软件包安装），ClickHouse以用户`clickhouse`的身份启动 . 因此，您需要在MySQL服务器中创建和配置此用户。
|
||||
|
||||
``` bash
|
||||
$ sudo mysql
|
||||
@ -57,7 +57,7 @@ mysql> CREATE USER 'clickhouse'@'localhost' IDENTIFIED BY 'clickhouse';
|
||||
mysql> GRANT ALL PRIVILEGES ON *.* TO 'clickhouse'@'clickhouse' WITH GRANT OPTION;
|
||||
```
|
||||
|
||||
然后配置连接 `/etc/odbc.ini`.
|
||||
然后在`/etc/odbc.ini`中配置连接 .
|
||||
|
||||
``` bash
|
||||
$ cat /etc/odbc.ini
|
||||
@ -70,7 +70,7 @@ USERNAME = clickhouse
|
||||
PASSWORD = clickhouse
|
||||
```
|
||||
|
||||
您可以使用 `isql` unixodbc安装中的实用程序。
|
||||
您可以从安装的unixodbc中使用 `isql` 实用程序来检查连接情况。
|
||||
|
||||
``` bash
|
||||
$ isql -v mysqlconn
|
||||
|
||||
@ -5,13 +5,13 @@ machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3
|
||||
|
||||
# 系统。data_type_families {#system_tables-data_type_families}
|
||||
|
||||
包含有关受支持的信息 [数据类型](../../sql-reference/data-types/).
|
||||
包含有关受支持的[数据类型](../../sql-reference/data-types/)的信息.
|
||||
|
||||
列:
|
||||
列字段包括:
|
||||
|
||||
- `name` ([字符串](../../sql-reference/data-types/string.md)) — Data type name.
|
||||
- `case_insensitive` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Property that shows whether you can use a data type name in a query in case insensitive manner or not. For example, `Date` 和 `date` 都是有效的。
|
||||
- `alias_to` ([字符串](../../sql-reference/data-types/string.md)) — Data type name for which `name` 是个化名
|
||||
- `name` ([String](../../sql-reference/data-types/string.md)) — 数据类型的名称.
|
||||
- `case_insensitive` ([UInt8](../../sql-reference/data-types/int-uint.md)) — 该属性显示是否可以在查询中以不区分大小写的方式使用数据类型名称。例如 `Date` 和 `date` 都是有效的。
|
||||
- `alias_to` ([String](../../sql-reference/data-types/string.md)) — 名称为别名的数据类型名称。
|
||||
|
||||
**示例**
|
||||
|
||||
@ -36,4 +36,4 @@ SELECT * FROM system.data_type_families WHERE alias_to = 'String'
|
||||
|
||||
**另请参阅**
|
||||
|
||||
- [语法](../../sql-reference/syntax.md) — Information about supported syntax.
|
||||
- [Syntax](../../sql-reference/syntax.md) — 关于所支持的语法信息.
|
||||
|
||||
@ -7,33 +7,33 @@ toc_title: "\u7CFB\u7EDF\u8868"
|
||||
|
||||
# 系统表 {#system-tables}
|
||||
|
||||
## 导言 {#system-tables-introduction}
|
||||
## 引言 {#system-tables-introduction}
|
||||
|
||||
系统表提供以下信息:
|
||||
系统表提供的信息如下:
|
||||
|
||||
- 服务器状态、进程和环境。
|
||||
- 服务器的状态、进程以及环境。
|
||||
- 服务器的内部进程。
|
||||
|
||||
系统表:
|
||||
|
||||
- 坐落于 `system` 数据库。
|
||||
- 仅适用于读取数据。
|
||||
- 不能删除或更改，但可以分离。
|
||||
- 存储于 `system` 数据库。
|
||||
- 仅提供数据读取功能。
|
||||
- 不能被删除或更改，但可以对其进行分离(detach)操作。
|
||||
|
||||
大多数系统表将数据存储在RAM中。 ClickHouse服务器在开始时创建此类系统表。
|
||||
大多数系统表将其数据存储在RAM中。 一个ClickHouse服务在刚启动时便会创建此类系统表。
|
||||
|
||||
与其他系统表不同，系统日志表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log), [part_log](../../operations/system-tables/part_log.md#system.part_log), crash_log and text_log 默认采用[MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 引擎并将其数据存储在存储文件系统中。 如果从文件系统中删除表，ClickHouse服务器会在下一次写入数据时再次创建空表。 如果系统表架构在新版本中发生更改，则ClickHouse会重命名当前表并创建一个新表。
|
||||
不同于其他系统表，系统日志表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log), [part_log](../../operations/system-tables/part_log.md#system.part_log), crash_log and text_log 默认采用[MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 引擎并将其数据存储在文件系统中。 如果人为的从文件系统中删除表，ClickHouse服务器会在下一次进行数据写入时再次创建空表。 如果系统表结构在新版本中发生更改，那么ClickHouse会重命名当前表并创建一个新表。
|
||||
|
||||
用户可以通过在`/etc/clickhouse-server/config.d/`下创建与系统表同名的配置文件, 或者在`/etc/clickhouse-server/config.xml`中设置相应配置项，来自定义系统日志表的结构。可以自定义的配置项如下:
|
||||
用户可以通过在`/etc/clickhouse-server/config.d/`下创建与系统表同名的配置文件, 或者在`/etc/clickhouse-server/config.xml`中设置相应配置项，来自定义系统日志表的结构。可供自定义的配置项如下:
|
||||
|
||||
- `database`: 系统日志表所在的数据库。这个选项目前已经废弃。所有的系统日表都位于`system`库中。
|
||||
- `table`: 系统日志表名。
|
||||
- `database`: 系统日志表所在的数据库。这个选项目前已经不推荐使用。所有的系统日表都位于`system`库中。
|
||||
- `table`: 接收数据写入的系统日志表。
|
||||
- `partition_by`: 指定[PARTITION BY](../../engines/table-engines/mergetree-family/custom-partitioning-key.md)表达式。
|
||||
- `ttl`: 指定系统日志表TTL选项。
|
||||
- `flush_interval_milliseconds`: 指定系统日志表数据落盘时间。
|
||||
- `engine`: 指定完整的表引擎定义。(以`ENGINE = `开始)。 这个选项与`partition_by`以及`ttl`冲突。如果两者一起设置，服务启动时会抛出异常并且退出。
|
||||
- `flush_interval_milliseconds`: 指定日志表数据刷新到磁盘的时间间隔。
|
||||
- `engine`: 指定完整的表引擎定义。(以`ENGINE = `开头)。 这个选项与`partition_by`以及`ttl`冲突。如果与两者一起设置，服务启动时会抛出异常并且退出。
|
||||
|
||||
一个配置定义的例子如下：
|
||||
配置定义的示例如下：
|
||||
|
||||
```
|
||||
<yandex>
|
||||
@ -50,20 +50,20 @@ toc_title: "\u7CFB\u7EDF\u8868"
|
||||
</yandex>
|
||||
```
|
||||
|
||||
默认情况下，表增长是无限的。 要控制表的大小，可以使用 TTL 删除过期日志记录的设置。 你也可以使用分区功能 `MergeTree`-发动机表。
|
||||
默认情况下，表增长是无限的。可以通过TTL 删除过期日志记录的设置来控制表的大小。 你也可以使用分区功能 `MergeTree`-引擎表。
|
||||
|
||||
## 系统指标的来源 {#system-tables-sources-of-system-metrics}
|
||||
|
||||
用于收集ClickHouse服务器使用的系统指标:
|
||||
|
||||
- `CAP_NET_ADMIN` 能力。
|
||||
- [procfs](https://en.wikipedia.org/wiki/Procfs) （仅在Linux中）。
|
||||
- [procfs](https://en.wikipedia.org/wiki/Procfs) （仅限于Linux）。
|
||||
|
||||
**procfs**
|
||||
|
||||
如果ClickHouse服务器没有 `CAP_NET_ADMIN` 能力，它试图回落到 `ProcfsMetricsProvider`. `ProcfsMetricsProvider` 允许收集每个查询系统指标（用于CPU和I/O）。
|
||||
如果ClickHouse服务器没有 `CAP_NET_ADMIN` 能力，那么它将试图退回到 `ProcfsMetricsProvider`. `ProcfsMetricsProvider` 允许收集每个查询系统指标（包括CPU和I/O）。
|
||||
|
||||
如果系统上支持并启用procfs，ClickHouse server将收集这些指标:
|
||||
如果系统上支持并启用procfs，ClickHouse server将收集如下指标:
|
||||
|
||||
- `OSCPUVirtualTimeMicroseconds`
|
||||
- `OSCPUWaitMicroseconds`
|
||||
|
||||
@ -238,7 +238,7 @@ SELECT a, b, c FROM (SELECT ...)
|
||||
|
||||
当一个`SELECT`子句包含`DISTINCT`, `GROUP BY`, `ORDER BY`, `LIMIT`时，请注意，这些仅会在插入数据时在每个单独的数据块上执行。例如，如果你在其中包含了`GROUP BY`，则只会在查询期间进行聚合，但聚合范围仅限于单个批的写入数据。数据不会进一步被聚合。但是当你使用一些其他数据聚合引擎时这是例外的，如：`SummingMergeTree`。
|
||||
|
||||
目前对物化视图执行`ALTER`是不支持的，因此这可能是不方便的。如果物化视图是使用的`TO [db.]name`的方式进行构建的，你可以使用`DETACH`语句现将视图剥离，然后使用`ALTER`运行在目标表上，然后使用`ATTACH`将之前剥离的表重新加载进来。
|
||||
目前对物化视图执行`ALTER`是不支持的，因此这可能是不方便的。如果物化视图是使用的`TO [db.]name`的方式进行构建的，你可以使用`DETACH`语句先将视图剥离，然后使用`ALTER`运行在目标表上，然后使用`ATTACH`将之前剥离的表重新加载进来。
|
||||
|
||||
视图看起来和普通的表相同。例如，你可以通过`SHOW TABLES`查看到它们。
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def')
|
||||
|
||||
含`INSERT INTO t VALUES` 的部分由完整SQL解析器处理，包含数据的部分 `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` 交给快速流式解析器解析。通过设置参数 [input_format_values_interpret_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions)，你也可以对数据部分开启完整SQL解析器。当 `input_format_values_interpret_expressions = 1` 时，CH优先采用快速流式解析器来解析数据。如果失败，CH再尝试用完整SQL解析器来处理，就像处理SQL [expression](#syntax-expressions) 一样。
|
||||
|
||||
数据可以采用任何格式。当CH接受到请求时，服务端先在内存中计算不超过 [max_query_size](../operations/settings/settings.md#settings-max_query_size) 字节的请求数据（默认1 mb），然后剩下部分交给快速流式解析器。
|
||||
数据可以采用任何格式。当CH接收到请求时，服务端先在内存中计算不超过 [max_query_size](../operations/settings/settings.md#settings-max_query_size) 字节的请求数据（默认1 mb），然后剩下部分交给快速流式解析器。
|
||||
|
||||
这将避免在处理大型的 `INSERT`语句时出现问题。
|
||||
|
||||
|
||||
@ -362,6 +362,20 @@
|
||||
bind_dn - template used to construct the DN to bind to.
|
||||
The resulting DN will be constructed by replacing all '{user_name}' substrings of the template with the actual
|
||||
user name during each authentication attempt.
|
||||
user_dn_detection - section with LDAP search parameters for detecting the actual user DN of the bound user.
|
||||
This is mainly used in search filters for further role mapping when the server is Active Directory. The
|
||||
resulting user DN will be used when replacing '{user_dn}' substrings wherever they are allowed. By default,
|
||||
user DN is set equal to bind DN, but once search is performed, it will be updated with to the actual detected
|
||||
user DN value.
|
||||
base_dn - template used to construct the base DN for the LDAP search.
|
||||
The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings
|
||||
of the template with the actual user name and bind DN during the LDAP search.
|
||||
scope - scope of the LDAP search.
|
||||
Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default).
|
||||
search_filter - template used to construct the search filter for the LDAP search.
|
||||
The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}'
|
||||
substrings of the template with the actual user name, bind DN, and base DN during the LDAP search.
|
||||
Note, that the special characters must be escaped properly in XML.
|
||||
verification_cooldown - a period of time, in seconds, after a successful bind attempt, during which a user will be assumed
|
||||
to be successfully authenticated for all consecutive requests without contacting the LDAP server.
|
||||
Specify 0 (the default) to disable caching and force contacting the LDAP server for each authentication request.
|
||||
@ -393,6 +407,17 @@
|
||||
<tls_ca_cert_dir>/path/to/tls_ca_cert_dir</tls_ca_cert_dir>
|
||||
<tls_cipher_suite>ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384</tls_cipher_suite>
|
||||
</my_ldap_server>
|
||||
Example (typical Active Directory with configured user DN detection for further role mapping):
|
||||
<my_ad_server>
|
||||
<host>localhost</host>
|
||||
<port>389</port>
|
||||
<bind_dn>EXAMPLE\{user_name}</bind_dn>
|
||||
<user_dn_detection>
|
||||
<base_dn>CN=Users,DC=example,DC=com</base_dn>
|
||||
<search_filter>(&(objectClass=user)(sAMAccountName={user_name}))</search_filter>
|
||||
</user_dn_detection>
|
||||
<enable_tls>no</enable_tls>
|
||||
</my_ad_server>
|
||||
-->
|
||||
</ldap_servers>
|
||||
|
||||
@ -444,15 +469,16 @@
|
||||
There can be multiple 'role_mapping' sections defined inside the same 'ldap' section. All of them will be
|
||||
applied.
|
||||
base_dn - template used to construct the base DN for the LDAP search.
|
||||
The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings
|
||||
of the template with the actual user name and bind DN during each LDAP search.
|
||||
The resulting DN will be constructed by replacing all '{user_name}', '{bind_dn}', and '{user_dn}'
|
||||
substrings of the template with the actual user name, bind DN, and user DN during each LDAP search.
|
||||
scope - scope of the LDAP search.
|
||||
Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default).
|
||||
search_filter - template used to construct the search filter for the LDAP search.
|
||||
The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}'
|
||||
substrings of the template with the actual user name, bind DN, and base DN during each LDAP search.
|
||||
The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', '{user_dn}', and
|
||||
'{base_dn}' substrings of the template with the actual user name, bind DN, user DN, and base DN during
|
||||
each LDAP search.
|
||||
Note, that the special characters must be escaped properly in XML.
|
||||
attribute - attribute name whose values will be returned by the LDAP search.
|
||||
attribute - attribute name whose values will be returned by the LDAP search. 'cn', by default.
|
||||
prefix - prefix, that will be expected to be in front of each string in the original list of strings returned by
|
||||
the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated
|
||||
as local role names. Empty, by default.
|
||||
@ -471,6 +497,17 @@
|
||||
<prefix>clickhouse_</prefix>
|
||||
</role_mapping>
|
||||
</ldap>
|
||||
Example (typical Active Directory with role mapping that relies on the detected user DN):
|
||||
<ldap>
|
||||
<server>my_ad_server</server>
|
||||
<role_mapping>
|
||||
<base_dn>CN=Users,DC=example,DC=com</base_dn>
|
||||
<attribute>CN</attribute>
|
||||
<scope>subtree</scope>
|
||||
<search_filter>(&(objectClass=group)(member={user_dn}))</search_filter>
|
||||
<prefix>clickhouse_</prefix>
|
||||
</role_mapping>
|
||||
</ldap>
|
||||
-->
|
||||
</user_directories>
|
||||
|
||||
|
||||
@ -143,11 +143,13 @@ ContextAccess::ContextAccess(const AccessControlManager & manager_, const Params
|
||||
: manager(&manager_)
|
||||
, params(params_)
|
||||
{
|
||||
std::lock_guard lock{mutex};
|
||||
|
||||
subscription_for_user_change = manager->subscribeForChanges(
|
||||
*params.user_id, [this](const UUID &, const AccessEntityPtr & entity)
|
||||
{
|
||||
UserPtr changed_user = entity ? typeid_cast<UserPtr>(entity) : nullptr;
|
||||
std::lock_guard lock{mutex};
|
||||
std::lock_guard lock2{mutex};
|
||||
setUser(changed_user);
|
||||
});
|
||||
|
||||
@ -189,7 +191,7 @@ void ContextAccess::setUser(const UserPtr & user_) const
|
||||
current_roles_with_admin_option = user->granted_roles.findGrantedWithAdminOption(params.current_roles);
|
||||
}
|
||||
|
||||
subscription_for_roles_changes = {};
|
||||
subscription_for_roles_changes.reset();
|
||||
enabled_roles = manager->getEnabledRoles(current_roles, current_roles_with_admin_option);
|
||||
subscription_for_roles_changes = enabled_roles->subscribeForChanges([this](const std::shared_ptr<const EnabledRolesInfo> & roles_info_)
|
||||
{
|
||||
|
||||
@ -20,13 +20,42 @@ namespace ErrorCodes
|
||||
namespace
|
||||
{
|
||||
|
||||
auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const String & name)
|
||||
void parseLDAPSearchParams(LDAPClient::SearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix)
|
||||
{
|
||||
const bool has_base_dn = config.has(prefix + ".base_dn");
|
||||
const bool has_search_filter = config.has(prefix + ".search_filter");
|
||||
const bool has_attribute = config.has(prefix + ".attribute");
|
||||
const bool has_scope = config.has(prefix + ".scope");
|
||||
|
||||
if (has_base_dn)
|
||||
params.base_dn = config.getString(prefix + ".base_dn");
|
||||
|
||||
if (has_search_filter)
|
||||
params.search_filter = config.getString(prefix + ".search_filter");
|
||||
|
||||
if (has_attribute)
|
||||
params.attribute = config.getString(prefix + ".attribute");
|
||||
|
||||
if (has_scope)
|
||||
{
|
||||
auto scope = config.getString(prefix + ".scope");
|
||||
boost::algorithm::to_lower(scope);
|
||||
|
||||
if (scope == "base") params.scope = LDAPClient::SearchParams::Scope::BASE;
|
||||
else if (scope == "one_level") params.scope = LDAPClient::SearchParams::Scope::ONE_LEVEL;
|
||||
else if (scope == "subtree") params.scope = LDAPClient::SearchParams::Scope::SUBTREE;
|
||||
else if (scope == "children") params.scope = LDAPClient::SearchParams::Scope::CHILDREN;
|
||||
else
|
||||
throw Exception("Invalid value for 'scope' field of LDAP search parameters in '" + prefix +
|
||||
"' section, must be one of 'base', 'one_level', 'subtree', or 'children'", ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
}
|
||||
|
||||
void parseLDAPServer(LDAPClient::Params & params, const Poco::Util::AbstractConfiguration & config, const String & name)
|
||||
{
|
||||
if (name.empty())
|
||||
throw Exception("LDAP server name cannot be empty", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
LDAPClient::Params params;
|
||||
|
||||
const String ldap_server_config = "ldap_servers." + name;
|
||||
|
||||
const bool has_host = config.has(ldap_server_config + ".host");
|
||||
@ -34,6 +63,7 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
|
||||
const bool has_bind_dn = config.has(ldap_server_config + ".bind_dn");
|
||||
const bool has_auth_dn_prefix = config.has(ldap_server_config + ".auth_dn_prefix");
|
||||
const bool has_auth_dn_suffix = config.has(ldap_server_config + ".auth_dn_suffix");
|
||||
const bool has_user_dn_detection = config.has(ldap_server_config + ".user_dn_detection");
|
||||
const bool has_verification_cooldown = config.has(ldap_server_config + ".verification_cooldown");
|
||||
const bool has_enable_tls = config.has(ldap_server_config + ".enable_tls");
|
||||
const bool has_tls_minimum_protocol_version = config.has(ldap_server_config + ".tls_minimum_protocol_version");
|
||||
@ -66,6 +96,17 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
|
||||
params.bind_dn = auth_dn_prefix + "{user_name}" + auth_dn_suffix;
|
||||
}
|
||||
|
||||
if (has_user_dn_detection)
|
||||
{
|
||||
if (!params.user_dn_detection)
|
||||
{
|
||||
params.user_dn_detection.emplace();
|
||||
params.user_dn_detection->attribute = "dn";
|
||||
}
|
||||
|
||||
parseLDAPSearchParams(*params.user_dn_detection, config, ldap_server_config + ".user_dn_detection");
|
||||
}
|
||||
|
||||
if (has_verification_cooldown)
|
||||
params.verification_cooldown = std::chrono::seconds{config.getUInt64(ldap_server_config + ".verification_cooldown")};
|
||||
|
||||
@ -143,14 +184,10 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
|
||||
}
|
||||
else
|
||||
params.port = (params.enable_tls == LDAPClient::Params::TLSEnable::YES ? 636 : 389);
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
auto parseKerberosParams(const Poco::Util::AbstractConfiguration & config)
|
||||
void parseKerberosParams(GSSAcceptorContext::Params & params, const Poco::Util::AbstractConfiguration & config)
|
||||
{
|
||||
GSSAcceptorContext::Params params;
|
||||
|
||||
Poco::Util::AbstractConfiguration::Keys keys;
|
||||
config.keys("kerberos", keys);
|
||||
|
||||
@ -180,12 +217,20 @@ auto parseKerberosParams(const Poco::Util::AbstractConfiguration & config)
|
||||
|
||||
params.realm = config.getString("kerberos.realm", "");
|
||||
params.principal = config.getString("kerberos.principal", "");
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void parseLDAPRoleSearchParams(LDAPClient::RoleSearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix)
|
||||
{
|
||||
parseLDAPSearchParams(params, config, prefix);
|
||||
|
||||
const bool has_prefix = config.has(prefix + ".prefix");
|
||||
|
||||
if (has_prefix)
|
||||
params.prefix = config.getString(prefix + ".prefix");
|
||||
}
|
||||
|
||||
void ExternalAuthenticators::reset()
|
||||
{
|
||||
std::scoped_lock lock(mutex);
|
||||
@ -229,7 +274,8 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur
|
||||
{
|
||||
try
|
||||
{
|
||||
ldap_client_params_blueprint.insert_or_assign(ldap_server_name, parseLDAPServer(config, ldap_server_name));
|
||||
ldap_client_params_blueprint.erase(ldap_server_name);
|
||||
parseLDAPServer(ldap_client_params_blueprint.emplace(ldap_server_name, LDAPClient::Params{}).first->second, config, ldap_server_name);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -240,7 +286,7 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur
|
||||
try
|
||||
{
|
||||
if (kerberos_keys_count > 0)
|
||||
kerberos_params = parseKerberosParams(config);
|
||||
parseKerberosParams(kerberos_params.emplace(), config);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -249,7 +295,7 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur
|
||||
}
|
||||
|
||||
bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const BasicCredentials & credentials,
|
||||
const LDAPClient::SearchParamsList * search_params, LDAPClient::SearchResultsList * search_results) const
|
||||
const LDAPClient::RoleSearchParamsList * role_search_params, LDAPClient::SearchResultsList * role_search_results) const
|
||||
{
|
||||
std::optional<LDAPClient::Params> params;
|
||||
std::size_t params_hash = 0;
|
||||
@ -267,9 +313,9 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B
|
||||
params->password = credentials.getPassword();
|
||||
|
||||
params->combineCoreHash(params_hash);
|
||||
if (search_params)
|
||||
if (role_search_params)
|
||||
{
|
||||
for (const auto & params_instance : *search_params)
|
||||
for (const auto & params_instance : *role_search_params)
|
||||
{
|
||||
params_instance.combineHash(params_hash);
|
||||
}
|
||||
@ -301,14 +347,14 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B
|
||||
|
||||
// Ensure that search_params are compatible.
|
||||
(
|
||||
search_params == nullptr ?
|
||||
entry.last_successful_search_results.empty() :
|
||||
search_params->size() == entry.last_successful_search_results.size()
|
||||
role_search_params == nullptr ?
|
||||
entry.last_successful_role_search_results.empty() :
|
||||
role_search_params->size() == entry.last_successful_role_search_results.size()
|
||||
)
|
||||
)
|
||||
{
|
||||
if (search_results)
|
||||
*search_results = entry.last_successful_search_results;
|
||||
if (role_search_results)
|
||||
*role_search_results = entry.last_successful_role_search_results;
|
||||
|
||||
return true;
|
||||
}
|
||||
@ -326,7 +372,7 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B
|
||||
}
|
||||
|
||||
LDAPSimpleAuthClient client(params.value());
|
||||
const auto result = client.authenticate(search_params, search_results);
|
||||
const auto result = client.authenticate(role_search_params, role_search_results);
|
||||
const auto current_check_timestamp = std::chrono::steady_clock::now();
|
||||
|
||||
// Update the cache, but only if this is the latest check and the server is still configured in a compatible way.
|
||||
@ -345,9 +391,9 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B
|
||||
|
||||
std::size_t new_params_hash = 0;
|
||||
new_params.combineCoreHash(new_params_hash);
|
||||
if (search_params)
|
||||
if (role_search_params)
|
||||
{
|
||||
for (const auto & params_instance : *search_params)
|
||||
for (const auto & params_instance : *role_search_params)
|
||||
{
|
||||
params_instance.combineHash(new_params_hash);
|
||||
}
|
||||
@ -363,17 +409,17 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B
|
||||
entry.last_successful_params_hash = params_hash;
|
||||
entry.last_successful_authentication_timestamp = current_check_timestamp;
|
||||
|
||||
if (search_results)
|
||||
entry.last_successful_search_results = *search_results;
|
||||
if (role_search_results)
|
||||
entry.last_successful_role_search_results = *role_search_results;
|
||||
else
|
||||
entry.last_successful_search_results.clear();
|
||||
entry.last_successful_role_search_results.clear();
|
||||
}
|
||||
else if (
|
||||
entry.last_successful_params_hash != params_hash ||
|
||||
(
|
||||
search_params == nullptr ?
|
||||
!entry.last_successful_search_results.empty() :
|
||||
search_params->size() != entry.last_successful_search_results.size()
|
||||
role_search_params == nullptr ?
|
||||
!entry.last_successful_role_search_results.empty() :
|
||||
role_search_params->size() != entry.last_successful_role_search_results.size()
|
||||
)
|
||||
)
|
||||
{
|
||||
|
||||
@ -34,7 +34,7 @@ public:
|
||||
|
||||
// The name and readiness of the credentials must be verified before calling these.
|
||||
bool checkLDAPCredentials(const String & server, const BasicCredentials & credentials,
|
||||
const LDAPClient::SearchParamsList * search_params = nullptr, LDAPClient::SearchResultsList * search_results = nullptr) const;
|
||||
const LDAPClient::RoleSearchParamsList * role_search_params = nullptr, LDAPClient::SearchResultsList * role_search_results = nullptr) const;
|
||||
bool checkKerberosCredentials(const String & realm, const GSSAcceptorContext & credentials) const;
|
||||
|
||||
GSSAcceptorContext::Params getKerberosParams() const;
|
||||
@ -44,7 +44,7 @@ private:
|
||||
{
|
||||
std::size_t last_successful_params_hash = 0;
|
||||
std::chrono::steady_clock::time_point last_successful_authentication_timestamp;
|
||||
LDAPClient::SearchResultsList last_successful_search_results;
|
||||
LDAPClient::SearchResultsList last_successful_role_search_results;
|
||||
};
|
||||
|
||||
using LDAPCache = std::unordered_map<String, LDAPCacheEntry>; // user name -> cache entry
|
||||
@ -58,4 +58,6 @@ private:
|
||||
std::optional<GSSAcceptorContext::Params> kerberos_params;
|
||||
};
|
||||
|
||||
void parseLDAPRoleSearchParams(LDAPClient::RoleSearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix);
|
||||
|
||||
}
|
||||
|
||||
@ -68,34 +68,15 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m
|
||||
common_roles_cfg.insert(role_names.begin(), role_names.end());
|
||||
}
|
||||
|
||||
LDAPClient::SearchParamsList role_search_params_cfg;
|
||||
LDAPClient::RoleSearchParamsList role_search_params_cfg;
|
||||
if (has_role_mapping)
|
||||
{
|
||||
Poco::Util::AbstractConfiguration::Keys all_keys;
|
||||
config.keys(prefix, all_keys);
|
||||
for (const auto & key : all_keys)
|
||||
{
|
||||
if (key != "role_mapping" && key.find("role_mapping[") != 0)
|
||||
continue;
|
||||
|
||||
const String rm_prefix = prefix_str + key;
|
||||
const String rm_prefix_str = rm_prefix + '.';
|
||||
role_search_params_cfg.emplace_back();
|
||||
auto & rm_params = role_search_params_cfg.back();
|
||||
|
||||
rm_params.base_dn = config.getString(rm_prefix_str + "base_dn", "");
|
||||
rm_params.search_filter = config.getString(rm_prefix_str + "search_filter", "");
|
||||
rm_params.attribute = config.getString(rm_prefix_str + "attribute", "cn");
|
||||
rm_params.prefix = config.getString(rm_prefix_str + "prefix", "");
|
||||
|
||||
auto scope = config.getString(rm_prefix_str + "scope", "subtree");
|
||||
boost::algorithm::to_lower(scope);
|
||||
if (scope == "base") rm_params.scope = LDAPClient::SearchParams::Scope::BASE;
|
||||
else if (scope == "one_level") rm_params.scope = LDAPClient::SearchParams::Scope::ONE_LEVEL;
|
||||
else if (scope == "subtree") rm_params.scope = LDAPClient::SearchParams::Scope::SUBTREE;
|
||||
else if (scope == "children") rm_params.scope = LDAPClient::SearchParams::Scope::CHILDREN;
|
||||
else
|
||||
throw Exception("Invalid value of 'scope' field in '" + key + "' section of LDAP user directory, must be one of 'base', 'one_level', 'subtree', or 'children'", ErrorCodes::BAD_ARGUMENTS);
|
||||
if (key == "role_mapping" || key.find("role_mapping[") == 0)
|
||||
parseLDAPRoleSearchParams(role_search_params_cfg.emplace_back(), config, prefix_str + key);
|
||||
}
|
||||
}
|
||||
|
||||
@ -364,7 +345,7 @@ std::set<String> LDAPAccessStorage::mapExternalRolesNoLock(const LDAPClient::Sea
|
||||
|
||||
|
||||
bool LDAPAccessStorage::areLDAPCredentialsValidNoLock(const User & user, const Credentials & credentials,
|
||||
const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & search_results) const
|
||||
const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & role_search_results) const
|
||||
{
|
||||
if (!credentials.isReady())
|
||||
return false;
|
||||
@ -373,7 +354,7 @@ bool LDAPAccessStorage::areLDAPCredentialsValidNoLock(const User & user, const C
|
||||
return false;
|
||||
|
||||
if (const auto * basic_credentials = dynamic_cast<const BasicCredentials *>(&credentials))
|
||||
return external_authenticators.checkLDAPCredentials(ldap_server_name, *basic_credentials, &role_search_params, &search_results);
|
||||
return external_authenticators.checkLDAPCredentials(ldap_server_name, *basic_credentials, &role_search_params, &role_search_results);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -68,12 +68,12 @@ private:
|
||||
void updateAssignedRolesNoLock(const UUID & id, const String & user_name, const LDAPClient::SearchResultsList & external_roles) const;
|
||||
std::set<String> mapExternalRolesNoLock(const LDAPClient::SearchResultsList & external_roles) const;
|
||||
bool areLDAPCredentialsValidNoLock(const User & user, const Credentials & credentials,
|
||||
const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & search_results) const;
|
||||
const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & role_search_results) const;
|
||||
|
||||
mutable std::recursive_mutex mutex;
|
||||
AccessControlManager * access_control_manager = nullptr;
|
||||
String ldap_server_name;
|
||||
LDAPClient::SearchParamsList role_search_params;
|
||||
LDAPClient::RoleSearchParamsList role_search_params;
|
||||
std::set<String> common_role_names; // role name that should be granted to all users at all times
|
||||
mutable std::map<String, std::size_t> external_role_hashes; // user name -> LDAPClient::SearchResultsList hash (most recently retrieved and processed)
|
||||
mutable std::map<String, std::set<String>> users_per_roles; // role name -> user names (...it should be granted to; may but don't have to exist for common roles)
|
||||
|
||||
@ -32,6 +32,11 @@ void LDAPClient::SearchParams::combineHash(std::size_t & seed) const
|
||||
boost::hash_combine(seed, static_cast<int>(scope));
|
||||
boost::hash_combine(seed, search_filter);
|
||||
boost::hash_combine(seed, attribute);
|
||||
}
|
||||
|
||||
void LDAPClient::RoleSearchParams::combineHash(std::size_t & seed) const
|
||||
{
|
||||
SearchParams::combineHash(seed);
|
||||
boost::hash_combine(seed, prefix);
|
||||
}
|
||||
|
||||
@ -42,6 +47,9 @@ void LDAPClient::Params::combineCoreHash(std::size_t & seed) const
|
||||
boost::hash_combine(seed, bind_dn);
|
||||
boost::hash_combine(seed, user);
|
||||
boost::hash_combine(seed, password);
|
||||
|
||||
if (user_dn_detection)
|
||||
user_dn_detection->combineHash(seed);
|
||||
}
|
||||
|
||||
LDAPClient::LDAPClient(const Params & params_)
|
||||
@ -286,18 +294,33 @@ void LDAPClient::openConnection()
|
||||
if (params.enable_tls == LDAPClient::Params::TLSEnable::YES_STARTTLS)
|
||||
diag(ldap_start_tls_s(handle, nullptr, nullptr));
|
||||
|
||||
final_user_name = escapeForLDAP(params.user);
|
||||
final_bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", final_user_name} });
|
||||
final_user_dn = final_bind_dn; // The default value... may be updated right after a successful bind.
|
||||
|
||||
switch (params.sasl_mechanism)
|
||||
{
|
||||
case LDAPClient::Params::SASLMechanism::SIMPLE:
|
||||
{
|
||||
const auto escaped_user_name = escapeForLDAP(params.user);
|
||||
const auto bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", escaped_user_name} });
|
||||
|
||||
::berval cred;
|
||||
cred.bv_val = const_cast<char *>(params.password.c_str());
|
||||
cred.bv_len = params.password.size();
|
||||
|
||||
diag(ldap_sasl_bind_s(handle, bind_dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr));
|
||||
diag(ldap_sasl_bind_s(handle, final_bind_dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr));
|
||||
|
||||
// Once bound, run the user DN search query and update the default value, if asked.
|
||||
if (params.user_dn_detection)
|
||||
{
|
||||
const auto user_dn_search_results = search(*params.user_dn_detection);
|
||||
|
||||
if (user_dn_search_results.empty())
|
||||
throw Exception("Failed to detect user DN: empty search results", ErrorCodes::LDAP_ERROR);
|
||||
|
||||
if (user_dn_search_results.size() > 1)
|
||||
throw Exception("Failed to detect user DN: more than one entry in the search results", ErrorCodes::LDAP_ERROR);
|
||||
|
||||
final_user_dn = *user_dn_search_results.begin();
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
@ -316,6 +339,9 @@ void LDAPClient::closeConnection() noexcept
|
||||
|
||||
ldap_unbind_ext_s(handle, nullptr, nullptr);
|
||||
handle = nullptr;
|
||||
final_user_name.clear();
|
||||
final_bind_dn.clear();
|
||||
final_user_dn.clear();
|
||||
}
|
||||
|
||||
LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params)
|
||||
@ -333,10 +359,19 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params)
|
||||
case SearchParams::Scope::CHILDREN: scope = LDAP_SCOPE_CHILDREN; break;
|
||||
}
|
||||
|
||||
const auto escaped_user_name = escapeForLDAP(params.user);
|
||||
const auto bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", escaped_user_name} });
|
||||
const auto base_dn = replacePlaceholders(search_params.base_dn, { {"{user_name}", escaped_user_name}, {"{bind_dn}", bind_dn} });
|
||||
const auto search_filter = replacePlaceholders(search_params.search_filter, { {"{user_name}", escaped_user_name}, {"{bind_dn}", bind_dn}, {"{base_dn}", base_dn} });
|
||||
const auto final_base_dn = replacePlaceholders(search_params.base_dn, {
|
||||
{"{user_name}", final_user_name},
|
||||
{"{bind_dn}", final_bind_dn},
|
||||
{"{user_dn}", final_user_dn}
|
||||
});
|
||||
|
||||
const auto final_search_filter = replacePlaceholders(search_params.search_filter, {
|
||||
{"{user_name}", final_user_name},
|
||||
{"{bind_dn}", final_bind_dn},
|
||||
{"{user_dn}", final_user_dn},
|
||||
{"{base_dn}", final_base_dn}
|
||||
});
|
||||
|
||||
char * attrs[] = { const_cast<char *>(search_params.attribute.c_str()), nullptr };
|
||||
::timeval timeout = { params.search_timeout.count(), 0 };
|
||||
LDAPMessage* msgs = nullptr;
|
||||
@ -349,7 +384,7 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params)
|
||||
}
|
||||
});
|
||||
|
||||
diag(ldap_search_ext_s(handle, base_dn.c_str(), scope, search_filter.c_str(), attrs, 0, nullptr, nullptr, &timeout, params.search_limit, &msgs));
|
||||
diag(ldap_search_ext_s(handle, final_base_dn.c_str(), scope, final_search_filter.c_str(), attrs, 0, nullptr, nullptr, &timeout, params.search_limit, &msgs));
|
||||
|
||||
for (
|
||||
auto * msg = ldap_first_message(handle, msgs);
|
||||
@ -361,6 +396,27 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params)
|
||||
{
|
||||
case LDAP_RES_SEARCH_ENTRY:
|
||||
{
|
||||
// Extract DN separately, if the requested attribute is DN.
|
||||
if (boost::iequals("dn", search_params.attribute))
|
||||
{
|
||||
BerElement * ber = nullptr;
|
||||
|
||||
SCOPE_EXIT({
|
||||
if (ber)
|
||||
{
|
||||
ber_free(ber, 0);
|
||||
ber = nullptr;
|
||||
}
|
||||
});
|
||||
|
||||
::berval bv;
|
||||
|
||||
diag(ldap_get_dn_ber(handle, msg, &ber, &bv));
|
||||
|
||||
if (bv.bv_val && bv.bv_len > 0)
|
||||
result.emplace(bv.bv_val, bv.bv_len);
|
||||
}
|
||||
|
||||
BerElement * ber = nullptr;
|
||||
|
||||
SCOPE_EXIT({
|
||||
@ -471,12 +527,12 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params)
|
||||
return result;
|
||||
}
|
||||
|
||||
bool LDAPSimpleAuthClient::authenticate(const SearchParamsList * search_params, SearchResultsList * search_results)
|
||||
bool LDAPSimpleAuthClient::authenticate(const RoleSearchParamsList * role_search_params, SearchResultsList * role_search_results)
|
||||
{
|
||||
if (params.user.empty())
|
||||
throw Exception("LDAP authentication of a user with empty name is not allowed", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
if (!search_params != !search_results)
|
||||
if (!role_search_params != !role_search_results)
|
||||
throw Exception("Cannot return LDAP search results", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
// Silently reject authentication attempt if the password is empty as if it didn't match.
|
||||
@ -489,21 +545,21 @@ bool LDAPSimpleAuthClient::authenticate(const SearchParamsList * search_params,
|
||||
openConnection();
|
||||
|
||||
// While connected, run search queries and save the results, if asked.
|
||||
if (search_params)
|
||||
if (role_search_params)
|
||||
{
|
||||
search_results->clear();
|
||||
search_results->reserve(search_params->size());
|
||||
role_search_results->clear();
|
||||
role_search_results->reserve(role_search_params->size());
|
||||
|
||||
try
|
||||
{
|
||||
for (const auto & single_search_params : *search_params)
|
||||
for (const auto & params_instance : *role_search_params)
|
||||
{
|
||||
search_results->emplace_back(search(single_search_params));
|
||||
role_search_results->emplace_back(search(params_instance));
|
||||
}
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
search_results->clear();
|
||||
role_search_results->clear();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@ -532,7 +588,7 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams &)
|
||||
throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME);
|
||||
}
|
||||
|
||||
bool LDAPSimpleAuthClient::authenticate(const SearchParamsList *, SearchResultsList *)
|
||||
bool LDAPSimpleAuthClient::authenticate(const RoleSearchParamsList *, SearchResultsList *)
|
||||
{
|
||||
throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME);
|
||||
}
|
||||
|
||||
@ -38,12 +38,20 @@ public:
|
||||
Scope scope = Scope::SUBTREE;
|
||||
String search_filter;
|
||||
String attribute = "cn";
|
||||
|
||||
void combineHash(std::size_t & seed) const;
|
||||
};
|
||||
|
||||
struct RoleSearchParams
|
||||
: public SearchParams
|
||||
{
|
||||
String prefix;
|
||||
|
||||
void combineHash(std::size_t & seed) const;
|
||||
};
|
||||
|
||||
using SearchParamsList = std::vector<SearchParams>;
|
||||
using RoleSearchParamsList = std::vector<RoleSearchParams>;
|
||||
|
||||
using SearchResults = std::set<String>;
|
||||
using SearchResultsList = std::vector<SearchResults>;
|
||||
|
||||
@ -105,6 +113,8 @@ public:
|
||||
String user;
|
||||
String password;
|
||||
|
||||
std::optional<SearchParams> user_dn_detection;
|
||||
|
||||
std::chrono::seconds verification_cooldown{0};
|
||||
|
||||
std::chrono::seconds operation_timeout{40};
|
||||
@ -134,6 +144,9 @@ protected:
|
||||
#if USE_LDAP
|
||||
LDAP * handle = nullptr;
|
||||
#endif
|
||||
String final_user_name;
|
||||
String final_bind_dn;
|
||||
String final_user_dn;
|
||||
};
|
||||
|
||||
class LDAPSimpleAuthClient
|
||||
@ -141,7 +154,7 @@ class LDAPSimpleAuthClient
|
||||
{
|
||||
public:
|
||||
using LDAPClient::LDAPClient;
|
||||
bool authenticate(const SearchParamsList * search_params, SearchResultsList * search_results);
|
||||
bool authenticate(const RoleSearchParamsList * role_search_params, SearchResultsList * role_search_results);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -106,8 +106,8 @@ endif()
|
||||
list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD})
|
||||
list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON})
|
||||
|
||||
list (APPEND dbms_sources Functions/IFunction.cpp Functions/IFunctionOld.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/replicate.cpp Functions/FunctionsLogical.cpp)
|
||||
list (APPEND dbms_headers Functions/IFunctionOld.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/replicate.h Functions/FunctionsLogical.h)
|
||||
list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/replicate.cpp Functions/FunctionsLogical.cpp)
|
||||
list (APPEND dbms_headers Functions/IFunction.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/replicate.h Functions/FunctionsLogical.h)
|
||||
|
||||
list (APPEND dbms_sources
|
||||
AggregateFunctions/AggregateFunctionFactory.cpp
|
||||
|
||||
@ -220,6 +220,12 @@ public:
|
||||
return find(key) != nullptr;
|
||||
}
|
||||
|
||||
Value & ALWAYS_INLINE operator[](const Key & key)
|
||||
{
|
||||
auto [it, _] = emplace(key);
|
||||
return it->getMapped();
|
||||
}
|
||||
|
||||
bool ALWAYS_INLINE erase(const Key & key)
|
||||
{
|
||||
auto key_hash = Base::hash(key);
|
||||
|
||||
@ -36,7 +36,7 @@ add_executable (arena_with_free_lists arena_with_free_lists.cpp)
|
||||
target_link_libraries (arena_with_free_lists PRIVATE dbms)
|
||||
|
||||
add_executable (lru_hash_map_perf lru_hash_map_perf.cpp)
|
||||
target_link_libraries (lru_hash_map_perf PRIVATE clickhouse_common_io)
|
||||
target_link_libraries (lru_hash_map_perf PRIVATE dbms)
|
||||
|
||||
add_executable (thread_creation_latency thread_creation_latency.cpp)
|
||||
target_link_libraries (thread_creation_latency PRIVATE clickhouse_common_io)
|
||||
|
||||
@ -7,23 +7,26 @@
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/HashTable/LRUHashMap.h>
|
||||
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
#include <Compression/CompressedReadBuffer.h>
|
||||
|
||||
template<class Key, class Value>
|
||||
class LRUHashMapBasic
|
||||
{
|
||||
public:
|
||||
using key_type = Key;
|
||||
using value_type = Value;
|
||||
using list_type = std::list<key_type>;
|
||||
using node = std::pair<value_type, typename list_type::iterator>;
|
||||
using map_type = std::unordered_map<key_type, node, DefaultHash<Key>>;
|
||||
using list_type = std::list<std::pair<key_type, value_type>>;
|
||||
using map_type = std::unordered_map<key_type, typename list_type::iterator>;
|
||||
|
||||
LRUHashMapBasic(size_t max_size_, bool preallocated)
|
||||
LRUHashMapBasic(size_t max_size_, bool preallocated = false)
|
||||
: hash_map(preallocated ? max_size_ : 32)
|
||||
, max_size(max_size_)
|
||||
{
|
||||
}
|
||||
|
||||
void insert(const Key &key, const Value &value)
|
||||
template<typename ...Args>
|
||||
std::pair<Value *, bool> emplace(const Key &key, Args &&... args)
|
||||
{
|
||||
auto it = hash_map.find(key);
|
||||
|
||||
@ -33,40 +36,39 @@ public:
|
||||
{
|
||||
auto iterator_to_remove = list.begin();
|
||||
|
||||
hash_map.erase(*iterator_to_remove);
|
||||
auto & key_to_remove = iterator_to_remove->first;
|
||||
hash_map.erase(key_to_remove);
|
||||
|
||||
list.erase(iterator_to_remove);
|
||||
}
|
||||
|
||||
list.push_back(key);
|
||||
hash_map[key] = std::make_pair(value, --list.end());
|
||||
|
||||
Value value(std::forward<Args>(args)...);
|
||||
auto node = std::make_pair(key, std::move(value));
|
||||
|
||||
list.push_back(std::move(node));
|
||||
|
||||
auto inserted_iterator = --list.end();
|
||||
|
||||
hash_map[key] = inserted_iterator;
|
||||
|
||||
return std::make_pair(&inserted_iterator->second, true);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto & [value_to_update, iterator_in_list_to_update] = it->second;
|
||||
auto & iterator_in_list_to_update = it->second;
|
||||
|
||||
list.splice(list.end(), list, iterator_in_list_to_update);
|
||||
iterator_in_list_to_update = --list.end();
|
||||
|
||||
iterator_in_list_to_update = list.end();
|
||||
value_to_update = value;
|
||||
return std::make_pair(&iterator_in_list_to_update->second, false);
|
||||
}
|
||||
}
|
||||
|
||||
value_type & get(const key_type &key)
|
||||
value_type & operator[](const key_type & key)
|
||||
{
|
||||
auto iterator_in_map = hash_map.find(key);
|
||||
assert(iterator_in_map != hash_map.end());
|
||||
|
||||
auto & [value_to_return, iterator_in_list_to_update] = iterator_in_map->second;
|
||||
|
||||
list.splice(list.end(), list, iterator_in_list_to_update);
|
||||
iterator_in_list_to_update = list.end();
|
||||
|
||||
return value_to_return;
|
||||
}
|
||||
|
||||
const value_type & get(const key_type & key) const
|
||||
{
|
||||
return const_cast<std::decay_t<decltype(*this)> *>(this)->get(key);
|
||||
auto [it, _] = emplace(key);
|
||||
return *it;
|
||||
}
|
||||
|
||||
size_t getMaxSize() const
|
||||
@ -101,110 +103,45 @@ private:
|
||||
size_t max_size;
|
||||
};
|
||||
|
||||
std::vector<UInt64> generateNumbersToInsert(size_t numbers_to_insert_size)
|
||||
template <typename Key, typename Map>
|
||||
static void NO_INLINE test(const Key * data, size_t size, const std::string & name)
|
||||
{
|
||||
std::vector<UInt64> numbers;
|
||||
numbers.reserve(numbers_to_insert_size);
|
||||
|
||||
std::random_device rd;
|
||||
pcg64 gen(rd());
|
||||
|
||||
UInt64 min = std::numeric_limits<UInt64>::min();
|
||||
UInt64 max = std::numeric_limits<UInt64>::max();
|
||||
|
||||
auto distribution = std::uniform_int_distribution<>(min, max);
|
||||
|
||||
for (size_t i = 0; i < numbers_to_insert_size; ++i)
|
||||
{
|
||||
UInt64 number = distribution(gen);
|
||||
numbers.emplace_back(number);
|
||||
}
|
||||
|
||||
return numbers;
|
||||
}
|
||||
|
||||
void testInsertElementsIntoHashMap(size_t map_size, const std::vector<UInt64> & numbers_to_insert, bool preallocated)
|
||||
{
|
||||
size_t numbers_to_insert_size = numbers_to_insert.size();
|
||||
std::cout << "TestInsertElementsIntoHashMap preallocated map size: " << map_size << " numbers to insert size: " << numbers_to_insert_size;
|
||||
std::cout << std::endl;
|
||||
|
||||
HashMap<int, int> hash_map(preallocated ? map_size : 32);
|
||||
|
||||
size_t cache_size = size / 10;
|
||||
Map cache(cache_size);
|
||||
Stopwatch watch;
|
||||
|
||||
for (size_t i = 0; i < numbers_to_insert_size; ++i)
|
||||
hash_map.insert({ numbers_to_insert[i], numbers_to_insert[i] });
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
++cache[data[i]];
|
||||
|
||||
std::cout << "Inserted in " << watch.elapsedMilliseconds() << " milliseconds" << std::endl;
|
||||
watch.stop();
|
||||
|
||||
UInt64 summ = 0;
|
||||
|
||||
for (size_t i = 0; i < numbers_to_insert_size; ++i)
|
||||
{
|
||||
auto * it = hash_map.find(numbers_to_insert[i]);
|
||||
|
||||
if (it)
|
||||
summ += it->getMapped();
|
||||
}
|
||||
|
||||
std::cout << "Calculated summ: " << summ << " in " << watch.elapsedMilliseconds() << " milliseconds" << std::endl;
|
||||
std::cerr << name
|
||||
<< ":\nElapsed: " << watch.elapsedSeconds()
|
||||
<< " (" << size / watch.elapsedSeconds() << " elem/sec.)"
|
||||
<< ", map size: " << cache.size() << "\n";
|
||||
}
|
||||
|
||||
void testInsertElementsIntoStandardMap(size_t map_size, const std::vector<UInt64> & numbers_to_insert, bool preallocated)
|
||||
template <typename Key>
|
||||
static void NO_INLINE testForType(size_t method, size_t rows_size)
|
||||
{
|
||||
size_t numbers_to_insert_size = numbers_to_insert.size();
|
||||
std::cout << "TestInsertElementsIntoStandardMap map size: " << map_size << " numbers to insert size: " << numbers_to_insert_size;
|
||||
std::cout << std::endl;
|
||||
std::cerr << std::fixed << std::setprecision(3);
|
||||
|
||||
std::unordered_map<int, int> hash_map(preallocated ? map_size : 32);
|
||||
std::vector<Key> data(rows_size);
|
||||
|
||||
Stopwatch watch;
|
||||
|
||||
for (size_t i = 0; i < numbers_to_insert_size; ++i)
|
||||
hash_map.insert({ numbers_to_insert[i], numbers_to_insert[i] });
|
||||
|
||||
std::cout << "Inserted in " << watch.elapsedMilliseconds() << " milliseconds" << std::endl;
|
||||
|
||||
UInt64 summ = 0;
|
||||
|
||||
for (size_t i = 0; i < numbers_to_insert_size; ++i)
|
||||
{
|
||||
auto it = hash_map.find(numbers_to_insert[i]);
|
||||
|
||||
if (it != hash_map.end())
|
||||
summ += it->second;
|
||||
DB::ReadBufferFromFileDescriptor in1(STDIN_FILENO);
|
||||
DB::CompressedReadBuffer in2(in1);
|
||||
in2.readStrict(reinterpret_cast<char*>(data.data()), sizeof(data[0]) * rows_size);
|
||||
}
|
||||
|
||||
std::cout << "Calculated summ: " << summ << " in " << watch.elapsedMilliseconds() << " milliseconds" << std::endl;
|
||||
}
|
||||
|
||||
template<typename LRUCache>
|
||||
UInt64 testInsertIntoEmptyCache(size_t map_size, const std::vector<UInt64> & numbers_to_insert, bool preallocated)
|
||||
{
|
||||
size_t numbers_to_insert_size = numbers_to_insert.size();
|
||||
std::cout << "Test testInsertPreallocated preallocated map size: " << map_size << " numbers to insert size: " << numbers_to_insert_size;
|
||||
std::cout << std::endl;
|
||||
|
||||
LRUCache cache(map_size, preallocated);
|
||||
Stopwatch watch;
|
||||
|
||||
for (size_t i = 0; i < numbers_to_insert_size; ++i)
|
||||
if (method == 0)
|
||||
{
|
||||
cache.insert(numbers_to_insert[i], numbers_to_insert[i]);
|
||||
test<Key, LRUHashMap<Key, UInt64>>(data.data(), data.size(), "CH HashMap");
|
||||
}
|
||||
else if (method == 1)
|
||||
{
|
||||
test<Key, LRUHashMapBasic<Key, UInt64>>(data.data(), data.size(), "BasicLRU");
|
||||
}
|
||||
|
||||
std::cout << "Inserted in " << watch.elapsedMilliseconds() << " milliseconds" << std::endl;
|
||||
|
||||
UInt64 summ = 0;
|
||||
|
||||
for (size_t i = 0; i < numbers_to_insert_size; ++i)
|
||||
if (cache.contains(numbers_to_insert[i]))
|
||||
summ += cache.get(numbers_to_insert[i]);
|
||||
|
||||
std::cout << "Calculated summ: " << summ << " in " << watch.elapsedMilliseconds() << " milliseconds" << std::endl;
|
||||
|
||||
return summ;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
@ -212,33 +149,34 @@ int main(int argc, char ** argv)
|
||||
(void)(argc);
|
||||
(void)(argv);
|
||||
|
||||
size_t hash_map_size = 1200000;
|
||||
size_t numbers_to_insert_size = 12000000;
|
||||
std::vector<UInt64> numbers = generateNumbersToInsert(numbers_to_insert_size);
|
||||
if (argc < 4)
|
||||
{
|
||||
std::cerr << "Usage: program method column_type_name rows_count < input_column.bin \n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::cout << "Test insert into HashMap preallocated=0" << std::endl;
|
||||
testInsertElementsIntoHashMap(hash_map_size, numbers, true);
|
||||
std::cout << std::endl;
|
||||
size_t method = std::stoull(argv[1]);
|
||||
std::string type_name = std::string(argv[2]);
|
||||
size_t n = std::stoull(argv[3]);
|
||||
|
||||
std::cout << "Test insert into HashMap preallocated=1" << std::endl;
|
||||
testInsertElementsIntoHashMap(hash_map_size, numbers, true);
|
||||
std::cout << std::endl;
|
||||
|
||||
std::cout << "Test LRUHashMap preallocated=0" << std::endl;
|
||||
testInsertIntoEmptyCache<LRUHashMap<UInt64, UInt64>>(hash_map_size, numbers, false);
|
||||
std::cout << std::endl;
|
||||
|
||||
std::cout << "Test LRUHashMap preallocated=1" << std::endl;
|
||||
testInsertIntoEmptyCache<LRUHashMap<UInt64, UInt64>>(hash_map_size, numbers, true);
|
||||
std::cout << std::endl;
|
||||
|
||||
std::cout << "Test LRUHashMapBasic preallocated=0" << std::endl;
|
||||
testInsertIntoEmptyCache<LRUHashMapBasic<UInt64, UInt64>>(hash_map_size, numbers, false);
|
||||
std::cout << std::endl;
|
||||
|
||||
std::cout << "Test LRUHashMapBasic preallocated=1" << std::endl;
|
||||
testInsertIntoEmptyCache<LRUHashMapBasic<UInt64, UInt64>>(hash_map_size, numbers, true);
|
||||
std::cout << std::endl;
|
||||
if (type_name == "UInt8")
|
||||
testForType<UInt8>(method, n);
|
||||
else if (type_name == "UInt16")
|
||||
testForType<UInt16>(method, n);
|
||||
else if (type_name == "UInt32")
|
||||
testForType<UInt32>(method, n);
|
||||
else if (type_name == "UInt64")
|
||||
testForType<UInt64>(method, n);
|
||||
else if (type_name == "Int8")
|
||||
testForType<Int8>(method, n);
|
||||
else if (type_name == "Int16")
|
||||
testForType<Int16>(method, n);
|
||||
else if (type_name == "Int32")
|
||||
testForType<Int32>(method, n);
|
||||
else if (type_name == "Int64")
|
||||
testForType<Int64>(method, n);
|
||||
else
|
||||
std::cerr << "Unexpected type passed " << type_name << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -1,29 +1,84 @@
|
||||
#include <Common/isLocalAddress.h>
|
||||
|
||||
#include <ifaddrs.h>
|
||||
#include <cstring>
|
||||
#include <optional>
|
||||
#include <common/types.h>
|
||||
#include <Poco/Util/Application.h>
|
||||
#include <Poco/Net/NetworkInterface.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Poco/Net/IPAddress.h>
|
||||
#include <Poco/Net/SocketAddress.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
bool isLocalAddress(const Poco::Net::IPAddress & address)
|
||||
namespace ErrorCodes
|
||||
{
|
||||
static auto interfaces = Poco::Net::NetworkInterface::list();
|
||||
extern const int SYSTEM_ERROR;
|
||||
}
|
||||
|
||||
return interfaces.end() != std::find_if(interfaces.begin(), interfaces.end(),
|
||||
[&] (const Poco::Net::NetworkInterface & interface)
|
||||
namespace
|
||||
{
|
||||
|
||||
struct NetworkInterfaces
|
||||
{
|
||||
ifaddrs * ifaddr;
|
||||
NetworkInterfaces()
|
||||
{
|
||||
if (getifaddrs(&ifaddr) == -1)
|
||||
{
|
||||
throwFromErrno("Cannot getifaddrs", ErrorCodes::SYSTEM_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
bool hasAddress(const Poco::Net::IPAddress & address) const
|
||||
{
|
||||
ifaddrs * iface;
|
||||
for (iface = ifaddr; iface != nullptr; iface = iface->ifa_next)
|
||||
{
|
||||
auto family = iface->ifa_addr->sa_family;
|
||||
std::optional<Poco::Net::IPAddress> interface_address;
|
||||
switch (family)
|
||||
{
|
||||
/// We interested only in IP-adresses
|
||||
case AF_INET:
|
||||
{
|
||||
interface_address.emplace(*(iface->ifa_addr));
|
||||
break;
|
||||
}
|
||||
case AF_INET6:
|
||||
{
|
||||
interface_address.emplace(&reinterpret_cast<const struct sockaddr_in6*>(iface->ifa_addr)->sin6_addr, sizeof(struct in6_addr));
|
||||
break;
|
||||
}
|
||||
default:
|
||||
continue;
|
||||
}
|
||||
|
||||
/** Compare the addresses without taking into account `scope`.
|
||||
* Theoretically, this may not be correct - depends on `route` setting
|
||||
* - through which interface we will actually access the specified address.
|
||||
*/
|
||||
return interface.address().length() == address.length()
|
||||
&& 0 == memcmp(interface.address().addr(), address.addr(), address.length());
|
||||
});
|
||||
if (interface_address->length() == address.length()
|
||||
&& 0 == memcmp(interface_address->addr(), address.addr(), address.length()))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
~NetworkInterfaces()
|
||||
{
|
||||
freeifaddrs(ifaddr);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
bool isLocalAddress(const Poco::Net::IPAddress & address)
|
||||
{
|
||||
NetworkInterfaces interfaces;
|
||||
return interfaces.hasAddress(address);
|
||||
}
|
||||
|
||||
bool isLocalAddress(const Poco::Net::SocketAddress & address, UInt16 clickhouse_port)
|
||||
|
||||
19
src/Common/tests/gtest_local_address.cpp
Normal file
19
src/Common/tests/gtest_local_address.cpp
Normal file
@ -0,0 +1,19 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <Common/isLocalAddress.h>
|
||||
#include <Common/ShellCommand.h>
|
||||
#include <Poco/Net/IPAddress.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
|
||||
|
||||
TEST(LocalAddress, SmokeTest)
|
||||
{
|
||||
auto cmd = DB::ShellCommand::executeDirect("/bin/hostname", {"-i"});
|
||||
std::string address_str;
|
||||
DB::readString(address_str, cmd->out);
|
||||
cmd->wait();
|
||||
std::cerr << "Got Address:" << address_str << std::endl;
|
||||
|
||||
Poco::Net::IPAddress address(address_str);
|
||||
|
||||
EXPECT_TRUE(DB::isLocalAddress(address));
|
||||
}
|
||||
@ -345,10 +345,12 @@ CodecTestSequence operator*(CodecTestSequence && left, T times)
|
||||
|
||||
std::ostream & operator<<(std::ostream & ostr, const Codec & codec)
|
||||
{
|
||||
return ostr << "Codec{"
|
||||
<< "name: " << codec.codec_statement
|
||||
<< ", expected_compression_ratio: " << *codec.expected_compression_ratio
|
||||
<< "}";
|
||||
ostr << "Codec{"
|
||||
<< "name: " << codec.codec_statement;
|
||||
if (codec.expected_compression_ratio)
|
||||
return ostr << ", expected_compression_ratio: " << *codec.expected_compression_ratio << "}";
|
||||
else
|
||||
return ostr << "}";
|
||||
}
|
||||
|
||||
std::ostream & operator<<(std::ostream & ostr, const CodecTestSequence & seq)
|
||||
|
||||
@ -35,6 +35,8 @@ public:
|
||||
|
||||
bool canBePromoted() const override { return false; }
|
||||
|
||||
bool canBeUsedAsVersion() const override { return true; }
|
||||
|
||||
protected:
|
||||
SerializationPtr doGetDefaultSerialization() const override;
|
||||
};
|
||||
|
||||
@ -35,6 +35,7 @@ public:
|
||||
bool textCanContainOnlyValidUTF8() const override { return true; }
|
||||
bool isComparable() const override { return true; }
|
||||
bool isValueUnambiguouslyRepresentedInContiguousMemoryRegion() const override { return true; }
|
||||
bool isValueUnambiguouslyRepresentedInFixedSizeContiguousMemoryRegion() const override { return true; }
|
||||
bool haveMaximumSizeOfValue() const override { return true; }
|
||||
size_t getSizeOfValueInMemory() const override { return sizeof(UUID); }
|
||||
bool isCategorial() const override { return true; }
|
||||
|
||||
@ -177,7 +177,7 @@ public:
|
||||
*/
|
||||
virtual bool canBeComparedWithCollation() const { return false; }
|
||||
|
||||
/** If the type is totally comparable (Ints, Date, DateTime, not nullable, not floats)
|
||||
/** If the type is totally comparable (Ints, Date, DateTime, DateTime64, not nullable, not floats)
|
||||
* and "simple" enough (not String, FixedString) to be used as version number
|
||||
* (to select rows with maximum version).
|
||||
*/
|
||||
|
||||
@ -203,7 +203,7 @@ void DatabaseLazy::shutdown()
|
||||
for (const auto & kv : tables_snapshot)
|
||||
{
|
||||
if (kv.second.table)
|
||||
kv.second.table->shutdown();
|
||||
kv.second.table->flushAndShutdown();
|
||||
}
|
||||
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
@ -529,7 +529,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep
|
||||
dropped_tables.push_back(tryGetTableUUID(table_name));
|
||||
dropped_dictionaries += table->isDictionary();
|
||||
|
||||
table->shutdown();
|
||||
table->flushAndShutdown();
|
||||
DatabaseAtomic::dropTable(getContext(), table_name, true);
|
||||
}
|
||||
else
|
||||
|
||||
@ -125,10 +125,15 @@ void DatabaseWithOwnTablesBase::shutdown()
|
||||
tables_snapshot = tables;
|
||||
}
|
||||
|
||||
for (const auto & kv : tables_snapshot)
|
||||
{
|
||||
kv.second->flush();
|
||||
}
|
||||
|
||||
for (const auto & kv : tables_snapshot)
|
||||
{
|
||||
auto table_id = kv.second->getStorageID();
|
||||
kv.second->shutdown();
|
||||
kv.second->flushAndShutdown();
|
||||
if (table_id.hasUUID())
|
||||
{
|
||||
assert(getDatabaseName() == DatabaseCatalog::TEMPORARY_DATABASE || getUUID() != UUIDHelpers::Nil);
|
||||
|
||||
@ -316,7 +316,7 @@ void DatabaseConnectionMySQL::shutdown()
|
||||
}
|
||||
|
||||
for (const auto & [table_name, modify_time_and_storage] : tables_snapshot)
|
||||
modify_time_and_storage.second->shutdown();
|
||||
modify_time_and_storage.second->flushAndShutdown();
|
||||
|
||||
std::lock_guard lock(mutex);
|
||||
local_tables_cache.clear();
|
||||
@ -343,7 +343,7 @@ void DatabaseConnectionMySQL::cleanOutdatedTables()
|
||||
{
|
||||
const auto table_lock = (*iterator)->lockExclusively(RWLockImpl::NO_QUERY, lock_acquire_timeout);
|
||||
|
||||
(*iterator)->shutdown();
|
||||
(*iterator)->flushAndShutdown();
|
||||
(*iterator)->is_dropped = true;
|
||||
iterator = outdated_tables.erase(iterator);
|
||||
}
|
||||
|
||||
@ -5,8 +5,8 @@ add_subdirectory(divide)
|
||||
include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake")
|
||||
add_headers_and_sources(clickhouse_functions .)
|
||||
|
||||
list(REMOVE_ITEM clickhouse_functions_sources IFunctionOld.cpp FunctionFactory.cpp FunctionHelpers.cpp)
|
||||
list(REMOVE_ITEM clickhouse_functions_headers IFunctionOld.h FunctionFactory.h FunctionHelpers.h)
|
||||
list(REMOVE_ITEM clickhouse_functions_sources IFunction.cpp FunctionFactory.cpp FunctionHelpers.cpp)
|
||||
list(REMOVE_ITEM clickhouse_functions_headers IFunction.h FunctionFactory.h FunctionHelpers.h)
|
||||
|
||||
add_library(clickhouse_functions ${clickhouse_functions_sources})
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/extractTimeZoneFromFunctionArguments.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <common/DateLUTImpl.h>
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnDecimal.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/extractTimeZoneFromFunctionArguments.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
|
||||
@ -23,7 +23,7 @@
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Columns/ColumnAggregateFunction.h>
|
||||
#include "Core/DecimalFunctions.h"
|
||||
#include "IFunctionOld.h"
|
||||
#include "IFunction.h"
|
||||
#include "FunctionHelpers.h"
|
||||
#include "IsOperation.h"
|
||||
#include "DivisionUtils.h"
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <ext/range.h>
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <Functions/CustomWeekTransforms.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/TransformDateTime64.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/castTypeToEither.h>
|
||||
#include <Functions/extractTimeZoneFromFunctionArguments.h>
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <Functions/extractTimeZoneFromFunctionArguments.h>
|
||||
#include <Functions/DateTimeTransforms.h>
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <common/getFQDNOrHostName.h>
|
||||
|
||||
@ -3,7 +3,6 @@
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
#include <Common/IFactoryWithAliases.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunctionAdaptors.h>
|
||||
|
||||
#include <functional>
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnFixedString.h>
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <DataTypes/Native.h>
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnDecimal.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnDecimal.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/GatherUtils/GatherUtils.h>
|
||||
#include <Functions/GatherUtils/Sources.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/PerformanceAdaptors.h>
|
||||
#include <Functions/TargetSpecific.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnFixedString.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnDecimal.h>
|
||||
#include <Columns/ColumnFixedString.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IsOperation.h>
|
||||
#include <Functions/castTypeToEither.h>
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Functions/extractTimeZoneFromFunctionArguments.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/assert_cast.h>
|
||||
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
#include <DataTypes/DataTypeUUID.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Common/IPv6ToBinary.h>
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/EmbeddedDictionaries.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Dictionaries/Embedded/RegionsHierarchy.h>
|
||||
#include <Dictionaries/Embedded/RegionsHierarchies.h>
|
||||
|
||||
@ -27,7 +27,7 @@
|
||||
#include <Interpreters/ExternalDictionariesLoader.h>
|
||||
#include <Interpreters/castColumn.h>
|
||||
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <ext/range.h>
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -39,7 +39,7 @@
|
||||
#include <Columns/ColumnFixedString.h>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/TargetSpecific.h>
|
||||
#include <Functions/PerformanceAdaptors.h>
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Core/AccurateComparison.h>
|
||||
#include <Functions/DummyJSONParser.h>
|
||||
#include <Functions/SimdJSONParser.h>
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
#include <Core/Defines.h>
|
||||
#include <DataTypes/IDataType.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <type_traits>
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <common/StringRef.h>
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <common/StringRef.h>
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <common/StringRef.h>
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/TargetSpecific.h>
|
||||
#include <Functions/PerformanceAdaptors.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Interpreters/castColumn.h>
|
||||
#include "IFunctionOld.h"
|
||||
#include "IFunction.h"
|
||||
#include <Common/intExp.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Core/Defines.h>
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/Regexps.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <common/StringRef.h>
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
#include <DataTypes/DataTypeFixedString.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Common/Volnitsky.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <IO/ReadBufferFromMemory.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
|
||||
@ -15,7 +15,17 @@
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <cstdlib>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include <Common/config.h>
|
||||
#endif
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
# pragma GCC diagnostic push
|
||||
# pragma GCC diagnostic ignored "-Wunused-parameter"
|
||||
# include <llvm/IR/IRBuilder.h>
|
||||
# pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -343,4 +353,77 @@ DataTypePtr IFunctionOverloadResolver::getReturnTypeWithoutLowCardinality(const
|
||||
return getReturnTypeImpl(arguments);
|
||||
}
|
||||
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
||||
static std::optional<DataTypes> removeNullables(const DataTypes & types)
|
||||
{
|
||||
for (const auto & type : types)
|
||||
{
|
||||
if (!typeid_cast<const DataTypeNullable *>(type.get()))
|
||||
continue;
|
||||
DataTypes filtered;
|
||||
for (const auto & sub_type : types)
|
||||
filtered.emplace_back(removeNullable(sub_type));
|
||||
return filtered;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
bool IFunction::isCompilable(const DataTypes & arguments) const
|
||||
{
|
||||
if (useDefaultImplementationForNulls())
|
||||
if (auto denulled = removeNullables(arguments))
|
||||
return isCompilableImpl(*denulled);
|
||||
return isCompilableImpl(arguments);
|
||||
}
|
||||
|
||||
llvm::Value * IFunction::compile(llvm::IRBuilderBase & builder, const DataTypes & arguments, Values values) const
|
||||
{
|
||||
auto denulled_arguments = removeNullables(arguments);
|
||||
if (useDefaultImplementationForNulls() && denulled_arguments)
|
||||
{
|
||||
auto & b = static_cast<llvm::IRBuilder<> &>(builder);
|
||||
|
||||
std::vector<llvm::Value*> unwrapped_values;
|
||||
std::vector<llvm::Value*> is_null_values;
|
||||
|
||||
unwrapped_values.reserve(arguments.size());
|
||||
is_null_values.reserve(arguments.size());
|
||||
|
||||
for (size_t i = 0; i < arguments.size(); ++i)
|
||||
{
|
||||
auto * value = values[i];
|
||||
|
||||
WhichDataType data_type(arguments[i]);
|
||||
if (data_type.isNullable())
|
||||
{
|
||||
unwrapped_values.emplace_back(b.CreateExtractValue(value, {0}));
|
||||
is_null_values.emplace_back(b.CreateExtractValue(value, {1}));
|
||||
}
|
||||
else
|
||||
{
|
||||
unwrapped_values.emplace_back(value);
|
||||
}
|
||||
}
|
||||
|
||||
auto * result = compileImpl(builder, *denulled_arguments, unwrapped_values);
|
||||
|
||||
auto * nullable_structure_type = toNativeType(b, makeNullable(getReturnTypeImpl(*denulled_arguments)));
|
||||
auto * nullable_structure_value = llvm::Constant::getNullValue(nullable_structure_type);
|
||||
|
||||
auto * nullable_structure_with_result_value = b.CreateInsertValue(nullable_structure_value, result, {0});
|
||||
auto * nullable_structure_result_null = b.CreateExtractValue(nullable_structure_with_result_value, {1});
|
||||
|
||||
for (auto * is_null_value : is_null_values)
|
||||
nullable_structure_result_null = b.CreateOr(nullable_structure_result_null, is_null_value);
|
||||
|
||||
return b.CreateInsertValue(nullable_structure_with_result_value, nullable_structure_result_null, {1});
|
||||
}
|
||||
|
||||
return compileImpl(builder, arguments, std::move(values));
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
@ -326,4 +326,120 @@ private:
|
||||
|
||||
using FunctionOverloadResolverPtr = std::shared_ptr<IFunctionOverloadResolver>;
|
||||
|
||||
/// Old function interface. Check documentation in IFunction.h.
|
||||
/// If client do not need statefull properties it can implement this interface.
|
||||
class IFunction
|
||||
{
|
||||
public:
|
||||
|
||||
virtual ~IFunction() = default;
|
||||
|
||||
virtual String getName() const = 0;
|
||||
|
||||
virtual ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const = 0;
|
||||
virtual ColumnPtr executeImplDryRun(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const
|
||||
{
|
||||
return executeImpl(arguments, result_type, input_rows_count);
|
||||
}
|
||||
|
||||
/** Default implementation in presence of Nullable arguments or NULL constants as arguments is the following:
|
||||
* if some of arguments are NULL constants then return NULL constant,
|
||||
* if some of arguments are Nullable, then execute function as usual for columns,
|
||||
* where Nullable columns are substituted with nested columns (they have arbitrary values in rows corresponding to NULL value)
|
||||
* and wrap result in Nullable column where NULLs are in all rows where any of arguments are NULL.
|
||||
*/
|
||||
virtual bool useDefaultImplementationForNulls() const { return true; }
|
||||
|
||||
/** If the function have non-zero number of arguments,
|
||||
* and if all arguments are constant, that we could automatically provide default implementation:
|
||||
* arguments are converted to ordinary columns with single value, then function is executed as usual,
|
||||
* and then the result is converted to constant column.
|
||||
*/
|
||||
virtual bool useDefaultImplementationForConstants() const { return false; }
|
||||
|
||||
/** If function arguments has single low cardinality column and all other arguments are constants, call function on nested column.
|
||||
* Otherwise, convert all low cardinality columns to ordinary columns.
|
||||
* Returns ColumnLowCardinality if at least one argument is ColumnLowCardinality.
|
||||
*/
|
||||
virtual bool useDefaultImplementationForLowCardinalityColumns() const { return true; }
|
||||
|
||||
/// If it isn't, will convert all ColumnLowCardinality arguments to full columns.
|
||||
virtual bool canBeExecutedOnLowCardinalityDictionary() const { return true; }
|
||||
|
||||
/** Some arguments could remain constant during this implementation.
|
||||
*/
|
||||
virtual ColumnNumbers getArgumentsThatAreAlwaysConstant() const { return {}; }
|
||||
|
||||
/** True if function can be called on default arguments (include Nullable's) and won't throw.
|
||||
* Counterexample: modulo(0, 0)
|
||||
*/
|
||||
virtual bool canBeExecutedOnDefaultArguments() const { return true; }
|
||||
|
||||
/// Properties from IFunctionBase (see IFunction.h)
|
||||
virtual bool isSuitableForConstantFolding() const { return true; }
|
||||
virtual ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & /*arguments*/) const { return nullptr; }
|
||||
virtual bool isInjective(const ColumnsWithTypeAndName & /*sample_columns*/) const { return false; }
|
||||
virtual bool isDeterministic() const { return true; }
|
||||
virtual bool isDeterministicInScopeOfQuery() const { return true; }
|
||||
virtual bool isStateful() const { return false; }
|
||||
virtual bool hasInformationAboutMonotonicity() const { return false; }
|
||||
|
||||
using Monotonicity = IFunctionBase::Monotonicity;
|
||||
virtual Monotonicity getMonotonicityForRange(const IDataType & /*type*/, const Field & /*left*/, const Field & /*right*/) const
|
||||
{
|
||||
throw Exception("Function " + getName() + " has no information about its monotonicity.", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
/// For non-variadic functions, return number of arguments; otherwise return zero (that should be ignored).
|
||||
virtual size_t getNumberOfArguments() const = 0;
|
||||
|
||||
virtual DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const
|
||||
{
|
||||
throw Exception("getReturnType is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
/// Get the result type by argument type. If the function does not apply to these arguments, throw an exception.
|
||||
virtual DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const
|
||||
{
|
||||
DataTypes data_types(arguments.size());
|
||||
for (size_t i = 0; i < arguments.size(); ++i)
|
||||
data_types[i] = arguments[i].type;
|
||||
|
||||
return getReturnTypeImpl(data_types);
|
||||
}
|
||||
|
||||
virtual bool isVariadic() const { return false; }
|
||||
|
||||
virtual void getLambdaArgumentTypes(DataTypes & /*arguments*/) const
|
||||
{
|
||||
throw Exception("Function " + getName() + " can't have lambda-expressions as arguments", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
|
||||
virtual ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const { return {}; }
|
||||
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
||||
bool isCompilable(const DataTypes & arguments) const;
|
||||
|
||||
llvm::Value * compile(llvm::IRBuilderBase &, const DataTypes & arguments, Values values) const;
|
||||
|
||||
#endif
|
||||
|
||||
protected:
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
||||
virtual bool isCompilableImpl(const DataTypes &) const { return false; }
|
||||
|
||||
virtual llvm::Value * compileImpl(llvm::IRBuilderBase &, const DataTypes &, Values) const
|
||||
{
|
||||
throw Exception(getName() + " is not JIT-compilable", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
#endif
|
||||
};
|
||||
|
||||
using FunctionPtr = std::shared_ptr<IFunction>;
|
||||
|
||||
}
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -1,91 +0,0 @@
|
||||
#include "IFunctionOld.h"
|
||||
|
||||
#include <DataTypes/Native.h>
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include <Common/config.h>
|
||||
#endif
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
# pragma GCC diagnostic push
|
||||
# pragma GCC diagnostic ignored "-Wunused-parameter"
|
||||
# include <llvm/IR/IRBuilder.h>
|
||||
# pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
||||
static std::optional<DataTypes> removeNullables(const DataTypes & types)
|
||||
{
|
||||
for (const auto & type : types)
|
||||
{
|
||||
if (!typeid_cast<const DataTypeNullable *>(type.get()))
|
||||
continue;
|
||||
DataTypes filtered;
|
||||
for (const auto & sub_type : types)
|
||||
filtered.emplace_back(removeNullable(sub_type));
|
||||
return filtered;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
bool IFunction::isCompilable(const DataTypes & arguments) const
|
||||
{
|
||||
if (useDefaultImplementationForNulls())
|
||||
if (auto denulled = removeNullables(arguments))
|
||||
return isCompilableImpl(*denulled);
|
||||
return isCompilableImpl(arguments);
|
||||
}
|
||||
|
||||
llvm::Value * IFunction::compile(llvm::IRBuilderBase & builder, const DataTypes & arguments, Values values) const
|
||||
{
|
||||
auto denulled_arguments = removeNullables(arguments);
|
||||
if (useDefaultImplementationForNulls() && denulled_arguments)
|
||||
{
|
||||
auto & b = static_cast<llvm::IRBuilder<> &>(builder);
|
||||
|
||||
std::vector<llvm::Value*> unwrapped_values;
|
||||
std::vector<llvm::Value*> is_null_values;
|
||||
|
||||
unwrapped_values.reserve(arguments.size());
|
||||
is_null_values.reserve(arguments.size());
|
||||
|
||||
for (size_t i = 0; i < arguments.size(); ++i)
|
||||
{
|
||||
auto * value = values[i];
|
||||
|
||||
WhichDataType data_type(arguments[i]);
|
||||
if (data_type.isNullable())
|
||||
{
|
||||
unwrapped_values.emplace_back(b.CreateExtractValue(value, {0}));
|
||||
is_null_values.emplace_back(b.CreateExtractValue(value, {1}));
|
||||
}
|
||||
else
|
||||
{
|
||||
unwrapped_values.emplace_back(value);
|
||||
}
|
||||
}
|
||||
|
||||
auto * result = compileImpl(builder, *denulled_arguments, unwrapped_values);
|
||||
|
||||
auto * nullable_structure_type = toNativeType(b, makeNullable(getReturnTypeImpl(*denulled_arguments)));
|
||||
auto * nullable_structure_value = llvm::Constant::getNullValue(nullable_structure_type);
|
||||
|
||||
auto * nullable_structure_with_result_value = b.CreateInsertValue(nullable_structure_value, result, {0});
|
||||
auto * nullable_structure_result_null = b.CreateExtractValue(nullable_structure_with_result_value, {1});
|
||||
|
||||
for (auto * is_null_value : is_null_values)
|
||||
nullable_structure_result_null = b.CreateOr(nullable_structure_result_null, is_null_value);
|
||||
|
||||
return b.CreateInsertValue(nullable_structure_with_result_value, nullable_structure_result_null, {1});
|
||||
}
|
||||
|
||||
return compileImpl(builder, arguments, std::move(values));
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
}
|
||||
@ -1,129 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <Functions/IFunction.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
/// Old function interface. Check documentation in IFunction.h
|
||||
class IFunction
|
||||
{
|
||||
public:
|
||||
|
||||
virtual ~IFunction() = default;
|
||||
|
||||
virtual String getName() const = 0;
|
||||
|
||||
virtual ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const = 0;
|
||||
virtual ColumnPtr executeImplDryRun(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const
|
||||
{
|
||||
return executeImpl(arguments, result_type, input_rows_count);
|
||||
}
|
||||
|
||||
/** Default implementation in presence of Nullable arguments or NULL constants as arguments is the following:
|
||||
* if some of arguments are NULL constants then return NULL constant,
|
||||
* if some of arguments are Nullable, then execute function as usual for columns,
|
||||
* where Nullable columns are substituted with nested columns (they have arbitrary values in rows corresponding to NULL value)
|
||||
* and wrap result in Nullable column where NULLs are in all rows where any of arguments are NULL.
|
||||
*/
|
||||
virtual bool useDefaultImplementationForNulls() const { return true; }
|
||||
|
||||
/** If the function have non-zero number of arguments,
|
||||
* and if all arguments are constant, that we could automatically provide default implementation:
|
||||
* arguments are converted to ordinary columns with single value, then function is executed as usual,
|
||||
* and then the result is converted to constant column.
|
||||
*/
|
||||
virtual bool useDefaultImplementationForConstants() const { return false; }
|
||||
|
||||
/** If function arguments has single low cardinality column and all other arguments are constants, call function on nested column.
|
||||
* Otherwise, convert all low cardinality columns to ordinary columns.
|
||||
* Returns ColumnLowCardinality if at least one argument is ColumnLowCardinality.
|
||||
*/
|
||||
virtual bool useDefaultImplementationForLowCardinalityColumns() const { return true; }
|
||||
|
||||
/// If it isn't, will convert all ColumnLowCardinality arguments to full columns.
|
||||
virtual bool canBeExecutedOnLowCardinalityDictionary() const { return true; }
|
||||
|
||||
/** Some arguments could remain constant during this implementation.
|
||||
*/
|
||||
virtual ColumnNumbers getArgumentsThatAreAlwaysConstant() const { return {}; }
|
||||
|
||||
/** True if function can be called on default arguments (include Nullable's) and won't throw.
|
||||
* Counterexample: modulo(0, 0)
|
||||
*/
|
||||
virtual bool canBeExecutedOnDefaultArguments() const { return true; }
|
||||
|
||||
/// Properties from IFunctionBase (see IFunction.h)
|
||||
virtual bool isSuitableForConstantFolding() const { return true; }
|
||||
virtual ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & /*arguments*/) const { return nullptr; }
|
||||
virtual bool isInjective(const ColumnsWithTypeAndName & /*sample_columns*/) const { return false; }
|
||||
virtual bool isDeterministic() const { return true; }
|
||||
virtual bool isDeterministicInScopeOfQuery() const { return true; }
|
||||
virtual bool isStateful() const { return false; }
|
||||
virtual bool hasInformationAboutMonotonicity() const { return false; }
|
||||
|
||||
using Monotonicity = IFunctionBase::Monotonicity;
|
||||
virtual Monotonicity getMonotonicityForRange(const IDataType & /*type*/, const Field & /*left*/, const Field & /*right*/) const
|
||||
{
|
||||
throw Exception("Function " + getName() + " has no information about its monotonicity.", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
/// For non-variadic functions, return number of arguments; otherwise return zero (that should be ignored).
|
||||
virtual size_t getNumberOfArguments() const = 0;
|
||||
|
||||
virtual DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const
|
||||
{
|
||||
throw Exception("getReturnType is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
/// Get the result type by argument type. If the function does not apply to these arguments, throw an exception.
|
||||
virtual DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const
|
||||
{
|
||||
DataTypes data_types(arguments.size());
|
||||
for (size_t i = 0; i < arguments.size(); ++i)
|
||||
data_types[i] = arguments[i].type;
|
||||
|
||||
return getReturnTypeImpl(data_types);
|
||||
}
|
||||
|
||||
virtual bool isVariadic() const { return false; }
|
||||
|
||||
virtual void getLambdaArgumentTypes(DataTypes & /*arguments*/) const
|
||||
{
|
||||
throw Exception("Function " + getName() + " can't have lambda-expressions as arguments", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
|
||||
virtual ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const { return {}; }
|
||||
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
||||
bool isCompilable(const DataTypes & arguments) const;
|
||||
|
||||
llvm::Value * compile(llvm::IRBuilderBase &, const DataTypes & arguments, Values values) const;
|
||||
|
||||
#endif
|
||||
|
||||
protected:
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
||||
virtual bool isCompilableImpl(const DataTypes &) const { return false; }
|
||||
|
||||
virtual llvm::Value * compileImpl(llvm::IRBuilderBase &, const DataTypes &, Values) const
|
||||
{
|
||||
throw Exception(getName() + " is not JIT-compilable", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
#endif
|
||||
};
|
||||
|
||||
using FunctionPtr = std::shared_ptr<IFunction>;
|
||||
|
||||
}
|
||||
@ -4,7 +4,7 @@
|
||||
#include <DataTypes/NumberTraits.h>
|
||||
#include <Interpreters/castColumn.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <ext/map.h>
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Functions/TargetSpecific.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <IO/WriteBufferFromArena.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Access/AccessFlags.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <ext/range.h>
|
||||
|
||||
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
#include <Columns/ColumnFunction.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/GatherUtils/GatherUtils.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#pragma once
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/getLeastSupertype.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Interpreters/AggregationCommon.h>
|
||||
#include <Common/ColumnsHashing.h>
|
||||
#include <Common/HashTable/ClearableHashMap.h>
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#pragma once
|
||||
#include <Functions/IFunctionOld.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user