mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-21 17:20:50 +00:00
Merge branch 'master' into feature/dotnet-integration-test
This commit is contained in:
mergify[bot]
GitHub
commit
742b50fe3f
@ -18,6 +18,8 @@
|
||||
|
||||
#define DATE_LUT_MAX (0xFFFFFFFFU - 86400)
|
||||
#define DATE_LUT_MAX_DAY_NUM 0xFFFF
|
||||
/// Max int value of Date32, DATE LUT cache size minus daynum_offset_epoch
|
||||
#define DATE_LUT_MAX_EXTEND_DAY_NUM (DATE_LUT_SIZE - 16436)
|
||||
|
||||
/// A constant to add to time_t so every supported time point becomes non-negative and still has the same remainder of division by 3600.
|
||||
/// If we treat "remainder of division" operation in the sense of modular arithmetic (not like in C++).
|
||||
@ -270,6 +272,8 @@ public:
|
||||
auto getOffsetAtStartOfEpoch() const { return offset_at_start_of_epoch; }
|
||||
auto getTimeOffsetAtStartOfLUT() const { return offset_at_start_of_lut; }
|
||||
|
||||
auto getDayNumOffsetEpoch() const { return daynum_offset_epoch; }
|
||||
|
||||
/// All functions below are thread-safe; arguments are not checked.
|
||||
|
||||
inline ExtendedDayNum toDayNum(ExtendedDayNum d) const
|
||||
@ -926,15 +930,17 @@ public:
|
||||
{
|
||||
if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31))
|
||||
return LUTIndex(0);
|
||||
|
||||
return LUTIndex{years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1};
|
||||
auto year_lut_index = (year - DATE_LUT_MIN_YEAR) * 12 + month - 1;
|
||||
UInt32 index = years_months_lut[year_lut_index].toUnderType() + day_of_month - 1;
|
||||
/// When date is out of range, default value is DATE_LUT_SIZE - 1 (2283-11-11)
|
||||
return LUTIndex{std::min(index, static_cast<UInt32>(DATE_LUT_SIZE - 1))};
|
||||
}
|
||||
|
||||
/// Create DayNum from year, month, day of month.
|
||||
inline ExtendedDayNum makeDayNum(Int16 year, UInt8 month, UInt8 day_of_month) const
|
||||
inline ExtendedDayNum makeDayNum(Int16 year, UInt8 month, UInt8 day_of_month, Int32 default_error_day_num = 0) const
|
||||
{
|
||||
if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31))
|
||||
return ExtendedDayNum(0);
|
||||
return ExtendedDayNum(default_error_day_num);
|
||||
|
||||
return toDayNum(makeLUTIndex(year, month, day_of_month));
|
||||
}
|
||||
@ -1091,9 +1097,9 @@ public:
|
||||
return lut[new_index].date + time;
|
||||
}
|
||||
|
||||
inline NO_SANITIZE_UNDEFINED Time addWeeks(Time t, Int64 delta) const
|
||||
inline NO_SANITIZE_UNDEFINED Time addWeeks(Time t, Int32 delta) const
|
||||
{
|
||||
return addDays(t, delta * 7);
|
||||
return addDays(t, static_cast<Int64>(delta) * 7);
|
||||
}
|
||||
|
||||
inline UInt8 saturateDayOfMonth(Int16 year, UInt8 month, UInt8 day_of_month) const
|
||||
@ -1158,14 +1164,14 @@ public:
|
||||
return toDayNum(addMonthsIndex(d, delta));
|
||||
}
|
||||
|
||||
inline Time NO_SANITIZE_UNDEFINED addQuarters(Time t, Int64 delta) const
|
||||
inline Time NO_SANITIZE_UNDEFINED addQuarters(Time t, Int32 delta) const
|
||||
{
|
||||
return addMonths(t, delta * 3);
|
||||
return addMonths(t, static_cast<Int64>(delta) * 3);
|
||||
}
|
||||
|
||||
inline ExtendedDayNum addQuarters(ExtendedDayNum d, Int64 delta) const
|
||||
inline ExtendedDayNum addQuarters(ExtendedDayNum d, Int32 delta) const
|
||||
{
|
||||
return addMonths(d, delta * 3);
|
||||
return addMonths(d, static_cast<Int64>(delta) * 3);
|
||||
}
|
||||
|
||||
template <typename DateOrTime>
|
||||
|
||||
@ -70,6 +70,14 @@ public:
|
||||
m_day = values.day_of_month;
|
||||
}
|
||||
|
||||
explicit LocalDate(ExtendedDayNum day_num)
|
||||
{
|
||||
const auto & values = DateLUT::instance().getValues(day_num);
|
||||
m_year = values.year;
|
||||
m_month = values.month;
|
||||
m_day = values.day_of_month;
|
||||
}
|
||||
|
||||
LocalDate(unsigned short year_, unsigned char month_, unsigned char day_)
|
||||
: m_year(year_), m_month(month_), m_day(day_)
|
||||
{
|
||||
@ -98,6 +106,12 @@ public:
|
||||
return DayNum(lut.makeDayNum(m_year, m_month, m_day).toUnderType());
|
||||
}
|
||||
|
||||
ExtendedDayNum getExtenedDayNum() const
|
||||
{
|
||||
const auto & lut = DateLUT::instance();
|
||||
return ExtendedDayNum (lut.makeDayNum(m_year, m_month, m_day).toUnderType());
|
||||
}
|
||||
|
||||
operator DayNum() const
|
||||
{
|
||||
return getDayNum();
|
||||
|
||||
@ -2,11 +2,11 @@
|
||||
|
||||
# NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
|
||||
# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
|
||||
SET(VERSION_REVISION 54453)
|
||||
SET(VERSION_REVISION 54454)
|
||||
SET(VERSION_MAJOR 21)
|
||||
SET(VERSION_MINOR 8)
|
||||
SET(VERSION_MINOR 9)
|
||||
SET(VERSION_PATCH 1)
|
||||
SET(VERSION_GITHASH fb895056568e26200629c7d19626e92d2dedc70d)
|
||||
SET(VERSION_DESCRIBE v21.8.1.1-prestable)
|
||||
SET(VERSION_STRING 21.8.1.1)
|
||||
SET(VERSION_GITHASH f48c5af90c2ad51955d1ee3b6b05d006b03e4238)
|
||||
SET(VERSION_DESCRIBE v21.9.1.1-prestable)
|
||||
SET(VERSION_STRING 21.9.1.1)
|
||||
# end of autochange
|
||||
|
||||
2
contrib/h3
vendored
2
contrib/h3
vendored
@ -1 +1 @@
|
||||
Subproject commit e209086ae1b5477307f545a0f6111780edc59940
|
||||
Subproject commit c7f46cfd71fb60e2fefc90e28abe81657deff735
|
||||
@ -3,21 +3,22 @@ set(H3_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/h3/src/h3lib")
|
||||
|
||||
set(SRCS
|
||||
"${H3_SOURCE_DIR}/lib/algos.c"
|
||||
"${H3_SOURCE_DIR}/lib/baseCells.c"
|
||||
"${H3_SOURCE_DIR}/lib/bbox.c"
|
||||
"${H3_SOURCE_DIR}/lib/coordijk.c"
|
||||
"${H3_SOURCE_DIR}/lib/faceijk.c"
|
||||
"${H3_SOURCE_DIR}/lib/geoCoord.c"
|
||||
"${H3_SOURCE_DIR}/lib/h3Index.c"
|
||||
"${H3_SOURCE_DIR}/lib/h3UniEdge.c"
|
||||
"${H3_SOURCE_DIR}/lib/linkedGeo.c"
|
||||
"${H3_SOURCE_DIR}/lib/localij.c"
|
||||
"${H3_SOURCE_DIR}/lib/mathExtensions.c"
|
||||
"${H3_SOURCE_DIR}/lib/bbox.c"
|
||||
"${H3_SOURCE_DIR}/lib/polygon.c"
|
||||
"${H3_SOURCE_DIR}/lib/h3Index.c"
|
||||
"${H3_SOURCE_DIR}/lib/vec2d.c"
|
||||
"${H3_SOURCE_DIR}/lib/vec3d.c"
|
||||
"${H3_SOURCE_DIR}/lib/vertex.c"
|
||||
"${H3_SOURCE_DIR}/lib/linkedGeo.c"
|
||||
"${H3_SOURCE_DIR}/lib/localij.c"
|
||||
"${H3_SOURCE_DIR}/lib/latLng.c"
|
||||
"${H3_SOURCE_DIR}/lib/directedEdge.c"
|
||||
"${H3_SOURCE_DIR}/lib/mathExtensions.c"
|
||||
"${H3_SOURCE_DIR}/lib/iterators.c"
|
||||
"${H3_SOURCE_DIR}/lib/vertexGraph.c"
|
||||
"${H3_SOURCE_DIR}/lib/faceijk.c"
|
||||
"${H3_SOURCE_DIR}/lib/baseCells.c"
|
||||
)
|
||||
|
||||
configure_file("${H3_SOURCE_DIR}/include/h3api.h.in" "${H3_BINARY_DIR}/include/h3api.h")
|
||||
|
||||
4
debian/changelog
vendored
4
debian/changelog
vendored
@ -1,5 +1,5 @@
|
||||
clickhouse (21.8.1.1) unstable; urgency=low
|
||||
clickhouse (21.9.1.1) unstable; urgency=low
|
||||
|
||||
* Modified source code
|
||||
|
||||
-- clickhouse-release <clickhouse-release@yandex-team.ru> Mon, 28 Jun 2021 00:50:15 +0300
|
||||
-- clickhouse-release <clickhouse-release@yandex-team.ru> Sat, 10 Jul 2021 08:22:49 +0300
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
|
||||
ARG version=21.8.1.*
|
||||
ARG version=21.9.1.*
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends \
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
|
||||
ARG version=21.8.1.*
|
||||
ARG version=21.9.1.*
|
||||
ARG gosu_ver=1.10
|
||||
|
||||
# set non-empty deb_location_url url to create a docker image
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
|
||||
ARG version=21.8.1.*
|
||||
ARG version=21.9.1.*
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y apt-transport-https dirmngr && \
|
||||
|
||||
@ -105,11 +105,11 @@ clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee te
|
||||
|
||||
5) ensure everything is correct, if the test output is incorrect (due to some bug for example), adjust the reference file using text editor.
|
||||
|
||||
#### How to create good test
|
||||
#### How to create a good test
|
||||
|
||||
- test should be
|
||||
- A test should be
|
||||
- minimal - create only tables related to tested functionality, remove unrelated columns and parts of query
|
||||
- fast - should not take longer than few seconds (better subseconds)
|
||||
- fast - should not take longer than a few seconds (better subseconds)
|
||||
- correct - fails then feature is not working
|
||||
- deterministic
|
||||
- isolated / stateless
|
||||
@ -126,6 +126,16 @@ clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee te
|
||||
- use other SQL files in the `0_stateless` folder as an example
|
||||
- ensure the feature / feature combination you want to test is not yet covered with existing tests
|
||||
|
||||
#### Test naming rules
|
||||
|
||||
It's important to name tests correctly, so one could turn some tests subset off in clickhouse-test invocation.
|
||||
|
||||
| Tester flag| What should be in test name | When flag should be added |
|
||||
|---|---|---|---|
|
||||
| `--[no-]zookeeper`| "zookeeper" or "replica" | Test uses tables from ReplicatedMergeTree family |
|
||||
| `--[no-]shard` | "shard" or "distributed" or "global"| Test using connections to 127.0.0.2 or similar |
|
||||
| `--[no-]long` | "long" or "deadlock" or "race" | Test runs longer than 60 seconds |
|
||||
|
||||
#### Commit / push / create PR.
|
||||
|
||||
1) commit & push your changes
|
||||
|
||||
@ -134,10 +134,10 @@ $ ./release
|
||||
|
||||
## Faster builds for development
|
||||
|
||||
Normally all tools of the ClickHouse bundle, such as `clickhouse-server`, `clickhouse-client` etc., are linked into a single static executable, `clickhouse`. This executable must be re-linked on every change, which might be slow. Two common ways to improve linking time are to use `lld` linker, and use the 'split' build configuration, which builds a separate binary for every tool, and further splits the code into several shared libraries. To enable these tweaks, pass the following flags to `cmake`:
|
||||
Normally all tools of the ClickHouse bundle, such as `clickhouse-server`, `clickhouse-client` etc., are linked into a single static executable, `clickhouse`. This executable must be re-linked on every change, which might be slow. One common way to improve build time is to use the 'split' build configuration, which builds a separate binary for every tool, and further splits the code into several shared libraries. To enable this tweak, pass the following flags to `cmake`:
|
||||
|
||||
```
|
||||
-DCMAKE_C_FLAGS="--ld-path=lld" -DCMAKE_CXX_FLAGS="--ld-path=lld" -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1
|
||||
-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1
|
||||
```
|
||||
|
||||
## You Don’t Have to Build ClickHouse {#you-dont-have-to-build-clickhouse}
|
||||
|
||||
114
docs/en/operations/clickhouse-keeper.md
Normal file
114
docs/en/operations/clickhouse-keeper.md
Normal file
@ -0,0 +1,114 @@
|
||||
---
|
||||
toc_priority: 66
|
||||
toc_title: ClickHouse Keeper
|
||||
---
|
||||
|
||||
# [pre-production] clickhouse-keeper
|
||||
|
||||
ClickHouse server use [ZooKeeper](https://zookeeper.apache.org/) coordination system for data [replication](../engines/table-engines/mergetree-family/replication.md) and [distributed DDL](../sql-reference/distributed-ddl.md) queries execution. ClickHouse Keeper is an alternative coordination system compatible with ZooKeeper.
|
||||
|
||||
!!! warning "Warning"
|
||||
This feature currently in pre-production stage. We test it in our CI and on small internal installations.
|
||||
|
||||
## Implemetation details
|
||||
|
||||
ZooKeeper is one of the first well-known open-source coordination systems. It's implemented in Java, has quite a simple and powerful data model. ZooKeeper's coordination algorithm called ZAB (ZooKeeper Atomic Broadcast) doesn't provide linearizability guarantees for reads, because each ZooKeeper node serves reads locally. Unlike ZooKeeper `clickhouse-keeper` written in C++ and use [RAFT algorithm](https://raft.github.io/) [implementation](https://github.com/eBay/NuRaft). This algorithm allows to have linearizability for reads and writes, has several open-source implementations in different languages.
|
||||
|
||||
By default, `clickhouse-keeper` provides the same guarantees as ZooKeeper (linearizable writes, non-linearizable reads). It has a compatible client-server protocol, so any standard ZooKeeper client can be used to interact with `clickhouse-keeper`. Snapshots and logs have an incompatible format with ZooKeeper, but `clickhouse-keeper-converter` tool allows to convert ZooKeeper data to `clickhouse-keeper` snapshot. Interserver protocol in `clickhouse-keeper` also incompatible with ZooKeeper so mixed ZooKeeper/clickhouse-keeper cluster is impossible.
|
||||
|
||||
## Configuration
|
||||
|
||||
`clickhouse-keeper` can be used as a standalone replacement for ZooKeeper or as an internal part of the `clickhouse-server`, but in both cases configuration is almost the same `.xml` file. The main `clickhouse-keeper` configuration tag is `<keeper_server>`. Keeper configuration has the following parameters:
|
||||
|
||||
- `tcp_port` — the port for a client to connect (default for ZooKeeper is `2181`)
|
||||
- `tcp_port_secure` — the secure port for a client to connect
|
||||
- `server_id` — unique server id, each participant of the clickhouse-keeper cluster must have a unique number (1, 2, 3, and so on)
|
||||
- `log_storage_path` — path to coordination logs, better to store logs on the non-busy device (same for ZooKeeper)
|
||||
- `snapshot_storage_path` — path to coordination snapshots
|
||||
|
||||
Other common parameters are inherited from clickhouse-server config (`listen_host`, `logger` and so on).
|
||||
|
||||
Internal coordination settings are located in `<keeper_server>.<coordination_settings>` section:
|
||||
|
||||
- `operation_timeout_ms` — timeout for a single client operation
|
||||
- `session_timeout_ms` — timeout for client session
|
||||
- `dead_session_check_period_ms` — how often clickhouse-keeper check dead sessions and remove them
|
||||
- `heart_beat_interval_ms` — how often a clickhouse-keeper leader will send heartbeats to followers
|
||||
- `election_timeout_lower_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it can initiate leader election
|
||||
- `election_timeout_upper_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it must initiate leader election
|
||||
- `rotate_log_storage_interval` — how many logs to store in a single file
|
||||
- `reserved_log_items` — how many coordination logs to store before compaction
|
||||
- `snapshot_distance` — how often clickhouse-keeper will create new snapshots (in the number of logs)
|
||||
- `snapshots_to_keep` — how many snapshots to keep
|
||||
- `stale_log_gap` — the threshold when leader consider follower as stale and send snapshot to it instead of logs
|
||||
- `force_sync` — call `fsync` on each write to coordination log
|
||||
- `raft_logs_level` — text logging level about coordination (trace, debug, and so on)
|
||||
- `shutdown_timeout` — wait to finish internal connections and shutdown
|
||||
- `startup_timeout` — if the server doesn't connect to other quorum participants in the specified timeout it will terminate
|
||||
|
||||
Quorum configuration is located in `<keeper_server>.<raft_configuration>` section and contain servers description. The only parameter for the whole quorum is `secure`, which enables encrypted connection for communication between quorum participants. The main parameters for each `<server>` are:
|
||||
|
||||
- `id` — server_id in quorum
|
||||
- `hostname` — hostname where this server placed
|
||||
- `port` — port where this server listen for connections
|
||||
|
||||
|
||||
Examples of configuration for quorum with three nodes can be found in [integration tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/integration) with `test_keeper_` prefix. Example configuration for server #1:
|
||||
|
||||
```xml
|
||||
<keeper_server>
|
||||
<tcp_port>2181</tcp_port>
|
||||
<server_id>1</server_id>
|
||||
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
||||
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
||||
|
||||
<coordination_settings>
|
||||
<operation_timeout_ms>10000</operation_timeout_ms>
|
||||
<session_timeout_ms>30000</session_timeout_ms>
|
||||
<raft_logs_level>trace</raft_logs_level>
|
||||
</coordination_settings>
|
||||
|
||||
<raft_configuration>
|
||||
<server>
|
||||
<id>1</id>
|
||||
<hostname>zoo1</hostname>
|
||||
<port>9444</port>
|
||||
</server>
|
||||
<server>
|
||||
<id>2</id>
|
||||
<hostname>zoo2</hostname>
|
||||
<port>9444</port>
|
||||
</server>
|
||||
<server>
|
||||
<id>3</id>
|
||||
<hostname>zoo3</hostname>
|
||||
<port>9444</port>
|
||||
</server>
|
||||
</raft_configuration>
|
||||
</keeper_server>
|
||||
```
|
||||
|
||||
## How to run
|
||||
|
||||
`clickhouse-keeper` is bundled into `clickhouse-server` package, just add configuration of `<keeper_server>` and start clickhouse-server as always. If you want to run standalone `clickhouse-keeper` you can start it in a similar way with:
|
||||
|
||||
```bash
|
||||
clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon
|
||||
```
|
||||
|
||||
## [experimental] Migration from ZooKeeper
|
||||
|
||||
Seamlessly migration from ZooKeeper to `clickhouse-keeper` is impossible you have to stop your ZooKeeper cluster, convert data and start `clickhouse-keeper`. `clickhouse-keeper-converter` tool allows to convert ZooKeeper logs and snapshots to `clickhouse-keeper` snapshot. It works only with ZooKeeper > 3.4. Steps for migration:
|
||||
|
||||
1. Stop all ZooKeeper nodes.
|
||||
|
||||
2. [optional, but recommended] Found ZooKeeper leader node, start and stop it again. It will force ZooKeeper to create consistent snapshot.
|
||||
|
||||
3. Run `clickhouse-keeper-converter` on leader, example
|
||||
|
||||
```bash
|
||||
clickhouse-keeper-converter --zookeeper-logs-dir /var/lib/zookeeper/version-2 --zookeeper-snapshots-dir /var/lib/zookeeper/version-2 --output-dir /path/to/clickhouse/keeper/snapshots
|
||||
```
|
||||
|
||||
4. Copy snapshot to `clickhouse-server` nodes with configured `keeper` or start `clickhouse-keeper` instead of ZooKeeper. Snapshot must persist only on leader node, leader will sync it automatically to other nodes.
|
||||
|
||||
@ -22,6 +22,23 @@ Some settings specified in the main configuration file can be overridden in othe
|
||||
|
||||
The config can also define “substitutions”. If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-include_from) element in the server config. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in `incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros](../operations/server-configuration-parameters/settings.md)).
|
||||
|
||||
If you want to replace an entire element with a substitution use `include` as element name.
|
||||
|
||||
XML substitution example:
|
||||
|
||||
```xml
|
||||
<yandex>
|
||||
<!-- Appends XML subtree found at `/profiles-in-zookeeper` ZK path to `<profiles>` element. -->
|
||||
<profiles from_zk="/profiles-in-zookeeper" />
|
||||
|
||||
<users>
|
||||
<!-- Replaces `include` element with the subtree found at `/users-in-zookeeper` ZK path. -->
|
||||
<include from_zk="/users-in-zookeeper" />
|
||||
<include from_zk="/other-users-in-zookeeper" />
|
||||
</users>
|
||||
</yandex>
|
||||
```
|
||||
|
||||
Substitutions can also be performed from ZooKeeper. To do this, specify the attribute `from_zk = "/path/to/node"`. The element value is replaced with the contents of the node at `/path/to/node` in ZooKeeper. You can also put an entire XML subtree on the ZooKeeper node and it will be fully inserted into the source element.
|
||||
|
||||
## User Settings {#user-settings}
|
||||
@ -32,6 +49,8 @@ Users configuration can be splitted into separate files similar to `config.xml`
|
||||
Directory name is defined as `users_config` setting without `.xml` postfix concatenated with `.d`.
|
||||
Directory `users.d` is used by default, as `users_config` defaults to `users.xml`.
|
||||
|
||||
Note that configuration files are first merged taking into account [Override](#override) settings and includes are processed after that.
|
||||
|
||||
## XML example {#example}
|
||||
|
||||
For example, you can have separate config file for each user like this:
|
||||
|
||||
@ -12,7 +12,7 @@ For information on connecting and configuring external dictionaries, see [Extern
|
||||
|
||||
## dictGet, dictGetOrDefault, dictGetOrNull {#dictget}
|
||||
|
||||
Retrieves values from an external dictionary.
|
||||
Retrieves values from an external dictionary.
|
||||
|
||||
``` sql
|
||||
dictGet('dict_name', attr_names, id_expr)
|
||||
@ -24,7 +24,7 @@ dictGetOrNull('dict_name', attr_name, id_expr)
|
||||
|
||||
- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `attr_names` — Name of the column of the dictionary, [String literal](../../sql-reference/syntax.md#syntax-string-literal), or tuple of column names, [Tuple](../../sql-reference/data-types/tuple.md)([String literal](../../sql-reference/syntax.md#syntax-string-literal)).
|
||||
- `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md) or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
|
||||
- `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning dictionary key-type value or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
|
||||
- `default_value_expr` — Values returned if the dictionary does not contain a row with the `id_expr` key. [Expression](../../sql-reference/syntax.md#syntax-expressions) or [Tuple](../../sql-reference/data-types/tuple.md)([Expression](../../sql-reference/syntax.md#syntax-expressions)), returning the value (or values) in the data types configured for the `attr_names` attribute.
|
||||
|
||||
**Returned value**
|
||||
@ -138,7 +138,7 @@ Configure the external dictionary:
|
||||
<name>c2</name>
|
||||
<type>String</type>
|
||||
<null_value></null_value>
|
||||
</attribute>
|
||||
</attribute>
|
||||
</structure>
|
||||
<lifetime>0</lifetime>
|
||||
</dictionary>
|
||||
@ -237,7 +237,7 @@ dictHas('dict_name', id_expr)
|
||||
**Arguments**
|
||||
|
||||
- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md) or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
|
||||
- `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning dictionary key-type value or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -292,16 +292,16 @@ Type: `UInt8`.
|
||||
|
||||
Returns first-level children as an array of indexes. It is the inverse transformation for [dictGetHierarchy](#dictgethierarchy).
|
||||
|
||||
**Syntax**
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
dictGetChildren(dict_name, key)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
**Arguments**
|
||||
|
||||
- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value.
|
||||
- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value.
|
||||
|
||||
**Returned values**
|
||||
|
||||
@ -339,7 +339,7 @@ SELECT dictGetChildren('hierarchy_flat_dictionary', number) FROM system.numbers
|
||||
|
||||
## dictGetDescendant {#dictgetdescendant}
|
||||
|
||||
Returns all descendants as if [dictGetChildren](#dictgetchildren) function was applied `level` times recursively.
|
||||
Returns all descendants as if [dictGetChildren](#dictgetchildren) function was applied `level` times recursively.
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -347,9 +347,9 @@ Returns all descendants as if [dictGetChildren](#dictgetchildren) function was a
|
||||
dictGetDescendants(dict_name, key, level)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
**Arguments**
|
||||
|
||||
- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value.
|
||||
- `level` — Hierarchy level. If `level = 0` returns all descendants to the end. [UInt8](../../sql-reference/data-types/int-uint.md).
|
||||
|
||||
|
||||
@ -87,6 +87,8 @@ Result:
|
||||
└───────┴───────┘
|
||||
```
|
||||
|
||||
Note: the names are implementation specific and are subject to change. You should not assume specific names of the columns after application of the `untuple`.
|
||||
|
||||
Example of using an `EXCEPT` expression:
|
||||
|
||||
Query:
|
||||
|
||||
@ -81,7 +81,7 @@ SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,
|
||||
**示例**
|
||||
|
||||
``` sql
|
||||
SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res
|
||||
SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res
|
||||
```
|
||||
|
||||
┌─res───────────────────────┐
|
||||
@ -174,7 +174,7 @@ SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS re
|
||||
│ [3] │
|
||||
└─────┘
|
||||
|
||||
## 位图 {#bitmapor}
|
||||
## 位图或 {#bitmapor}
|
||||
|
||||
为两个位图对象进行或操作，返回一个新的位图对象。
|
||||
|
||||
|
||||
@ -1,13 +1,8 @@
|
||||
---
|
||||
machine_translated: true
|
||||
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
|
||||
toc_priority: 42
|
||||
toc_title: mysql
|
||||
---
|
||||
|
||||
# mysql {#mysql}
|
||||
|
||||
允许 `SELECT` 要对存储在远程MySQL服务器上的数据执行的查询。
|
||||
允许对存储在远程MySQL服务器上的数据执行`SELECT`和`INSERT`查询。
|
||||
|
||||
**语法**
|
||||
|
||||
``` sql
|
||||
mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']);
|
||||
@ -15,31 +10,44 @@ mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_
|
||||
|
||||
**参数**
|
||||
|
||||
- `host:port` — MySQL server address.
|
||||
- `host:port` — MySQL服务器地址.
|
||||
|
||||
- `database` — Remote database name.
|
||||
- `database` — 远程数据库名称.
|
||||
|
||||
- `table` — Remote table name.
|
||||
- `table` — 远程表名称.
|
||||
|
||||
- `user` — MySQL user.
|
||||
- `user` — MySQL用户.
|
||||
|
||||
- `password` — User password.
|
||||
- `password` — 用户密码.
|
||||
|
||||
- `replace_query` — Flag that converts `INSERT INTO` 查询到 `REPLACE INTO`. 如果 `replace_query=1`，查询被替换。
|
||||
- `replace_query` — 将INSERT INTO` 查询转换为 `REPLACE INTO`的标志。如果 `replace_query=1`，查询被替换。
|
||||
|
||||
- `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` 表达式被添加到 `INSERT` 查询。
|
||||
- `on_duplicate_clause` — 添加 `ON DUPLICATE KEY on_duplicate_clause` 表达式到 `INSERT` 查询。明确规定只能使用 `replace_query = 0` ，如果你同时设置replace_query = 1`和`on_duplicate_clause`，ClickHouse将产生异常。
|
||||
|
||||
Example: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, where `on_duplicate_clause` is `UPDATE c2 = c2 + 1`. See the MySQL documentation to find which `on_duplicate_clause` you can use with the `ON DUPLICATE KEY` clause.
|
||||
示例：`INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`
|
||||
|
||||
To specify `on_duplicate_clause` you need to pass `0` to the `replace_query` parameter. If you simultaneously pass `replace_query = 1` and `on_duplicate_clause`, ClickHouse generates an exception.
|
||||
`on_duplicate_clause`这里是`UPDATE c2 = c2 + 1`。请查阅MySQL文档，来找到可以和`ON DUPLICATE KEY`一起使用的 `on_duplicate_clause`子句。
|
||||
|
||||
简单 `WHERE` 条款如 `=, !=, >, >=, <, <=` 当前在MySQL服务器上执行。
|
||||
简单的 `WHERE` 子句如 `=, !=, >, >=, <, <=` 将即时在MySQL服务器上执行。其余的条件和 `LIMIT` 只有在对MySQL的查询完成后，才会在ClickHouse中执行采样约束。
|
||||
|
||||
其余的条件和 `LIMIT` 只有在对MySQL的查询完成后，才会在ClickHouse中执行采样约束。
|
||||
支持使用`|`并列进行多副本查询，示例如下：
|
||||
|
||||
```sql
|
||||
SELECT name FROM mysql(`mysql{1|2|3}:3306`, 'mysql_database', 'mysql_table', 'user', 'password');
|
||||
```
|
||||
|
||||
或
|
||||
|
||||
```sql
|
||||
SELECT name FROM mysql(`mysql1:3306|mysql2:3306|mysql3:3306`, 'mysql_database', 'mysql_table', 'user', 'password');
|
||||
```
|
||||
|
||||
**返回值**
|
||||
|
||||
与原始MySQL表具有相同列的table对象。
|
||||
与原始MySQL表具有相同列的表对象。
|
||||
|
||||
!!! note "注意"
|
||||
在`INSERT`查询中为了区分`mysql(...)`与带有列名列表的表名的表函数，你必须使用关键字`FUNCTION`或`TABLE FUNCTION`。查看如下示例。
|
||||
|
||||
## 用法示例 {#usage-example}
|
||||
|
||||
@ -66,7 +74,7 @@ mysql> select * from test;
|
||||
1 row in set (0,00 sec)
|
||||
```
|
||||
|
||||
从ClickHouse中选择数据:
|
||||
从ClickHouse中查询数据:
|
||||
|
||||
``` sql
|
||||
SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123')
|
||||
@ -78,6 +86,21 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123')
|
||||
└────────┴──────────────┴───────┴────────────────┘
|
||||
```
|
||||
|
||||
替换和插入：
|
||||
|
||||
```sql
|
||||
INSERT INTO FUNCTION mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 1) (int_id, float) VALUES (1, 3);
|
||||
INSERT INTO TABLE FUNCTION mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 0, 'UPDATE int_id = int_id + 1') (int_id, float) VALUES (1, 4);
|
||||
SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123');
|
||||
```
|
||||
|
||||
```text
|
||||
┌─int_id─┬─float─┐
|
||||
│ 1 │ 3 │
|
||||
│ 2 │ 4 │
|
||||
└────────┴───────┘
|
||||
```
|
||||
|
||||
## 另请参阅 {#see-also}
|
||||
|
||||
- [该 ‘MySQL’ 表引擎](../../engines/table-engines/integrations/mysql.md)
|
||||
|
||||
@ -430,6 +430,7 @@ private:
|
||||
{TokenType::ClosingRoundBracket, Replxx::Color::BROWN},
|
||||
{TokenType::OpeningSquareBracket, Replxx::Color::BROWN},
|
||||
{TokenType::ClosingSquareBracket, Replxx::Color::BROWN},
|
||||
{TokenType::DoubleColon, Replxx::Color::BROWN},
|
||||
{TokenType::OpeningCurlyBrace, Replxx::Color::INTENSE},
|
||||
{TokenType::ClosingCurlyBrace, Replxx::Color::INTENSE},
|
||||
|
||||
|
||||
@ -388,24 +388,32 @@ void LocalServer::processQueries()
|
||||
/// Use the same query_id (and thread group) for all queries
|
||||
CurrentThread::QueryScope query_scope_holder(context);
|
||||
|
||||
///Set progress show
|
||||
/// Set progress show
|
||||
need_render_progress = config().getBool("progress", false);
|
||||
|
||||
std::function<void()> finalize_progress;
|
||||
if (need_render_progress)
|
||||
{
|
||||
/// Set progress callback, which can be run from multiple threads.
|
||||
context->setProgressCallback([&](const Progress & value)
|
||||
{
|
||||
/// Write progress only if progress was updated
|
||||
if (progress_indication.updateProgress(value))
|
||||
progress_indication.writeProgress();
|
||||
});
|
||||
|
||||
/// Set finalizing callback for progress, which is called right before finalizing query output.
|
||||
finalize_progress = [&]()
|
||||
{
|
||||
progress_indication.clearProgressOutput();
|
||||
};
|
||||
|
||||
/// Set callback for file processing progress.
|
||||
progress_indication.setFileProgressCallback(context);
|
||||
}
|
||||
|
||||
bool echo_queries = config().hasOption("echo") || config().hasOption("verbose");
|
||||
|
||||
if (need_render_progress)
|
||||
progress_indication.setFileProgressCallback(context);
|
||||
|
||||
std::exception_ptr exception;
|
||||
|
||||
for (const auto & query : queries)
|
||||
@ -425,7 +433,7 @@ void LocalServer::processQueries()
|
||||
|
||||
try
|
||||
{
|
||||
executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, context, {});
|
||||
executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, context, {}, finalize_progress);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
|
||||
@ -477,17 +477,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::getVersionRevision());
|
||||
CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
|
||||
|
||||
if (ThreadFuzzer::instance().isEffective())
|
||||
LOG_WARNING(log, "ThreadFuzzer is enabled. Application will run slowly and unstable.");
|
||||
|
||||
#if !defined(NDEBUG) || !defined(__OPTIMIZE__)
|
||||
LOG_WARNING(log, "Server was built in debug mode. It will work slowly.");
|
||||
#endif
|
||||
|
||||
#if defined(SANITIZER)
|
||||
LOG_WARNING(log, "Server was built with sanitizer. It will work slowly.");
|
||||
#endif
|
||||
|
||||
/** Context contains all that query execution is dependent:
|
||||
* settings, available functions, data types, aggregate functions, databases, ...
|
||||
*/
|
||||
@ -497,6 +486,18 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
global_context->makeGlobalContext();
|
||||
global_context->setApplicationType(Context::ApplicationType::SERVER);
|
||||
|
||||
#if !defined(NDEBUG) || !defined(__OPTIMIZE__)
|
||||
global_context->addWarningMessage("Server was built in debug mode. It will work slowly.");
|
||||
#endif
|
||||
|
||||
if (ThreadFuzzer::instance().isEffective())
|
||||
global_context->addWarningMessage("ThreadFuzzer is enabled. Application will run slowly and unstable.");
|
||||
|
||||
#if defined(SANITIZER)
|
||||
global_context->addWarningMessage("Server was built with sanitizer. It will work slowly.");
|
||||
#endif
|
||||
|
||||
|
||||
// Initialize global thread pool. Do it before we fetch configs from zookeeper
|
||||
// nodes (`from_zk`), because ZooKeeper interface uses the pool. We will
|
||||
// ignore `max_thread_pool_size` in configs we fetch from ZK, but oh well.
|
||||
@ -552,8 +553,10 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
if (ptrace(PTRACE_TRACEME, 0, nullptr, nullptr) == -1)
|
||||
{
|
||||
/// Program is run under debugger. Modification of it's binary image is ok for breakpoints.
|
||||
LOG_WARNING(log, "Server is run under debugger and its binary image is modified (most likely with breakpoints).",
|
||||
calculated_binary_hash);
|
||||
global_context->addWarningMessage(
|
||||
fmt::format("Server is run under debugger and its binary image is modified (most likely with breakpoints).",
|
||||
calculated_binary_hash)
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -636,7 +639,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_WARNING(log, message);
|
||||
global_context->addWarningMessage(message);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -9,6 +9,14 @@
|
||||
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include <Common/config.h>
|
||||
#endif
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
# include <llvm/IR/IRBuilder.h>
|
||||
# include <DataTypes/Native.h>
|
||||
#endif
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -21,6 +29,21 @@ struct AggregateFunctionGroupBitOrData
|
||||
T value = 0;
|
||||
static const char * name() { return "groupBitOr"; }
|
||||
void update(T x) { value |= x; }
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
||||
static void compileCreate(llvm::IRBuilderBase & builder, llvm::Value * value_ptr)
|
||||
{
|
||||
auto type = toNativeType<T>(builder);
|
||||
builder.CreateStore(llvm::Constant::getNullValue(type), value_ptr);
|
||||
}
|
||||
|
||||
static llvm::Value* compileUpdate(llvm::IRBuilderBase & builder, llvm::Value * lhs, llvm::Value * rhs)
|
||||
{
|
||||
return builder.CreateOr(lhs, rhs);
|
||||
}
|
||||
|
||||
#endif
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
@ -29,6 +52,21 @@ struct AggregateFunctionGroupBitAndData
|
||||
T value = -1; /// Two's complement arithmetic, sign extension.
|
||||
static const char * name() { return "groupBitAnd"; }
|
||||
void update(T x) { value &= x; }
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
||||
static void compileCreate(llvm::IRBuilderBase & builder, llvm::Value * value_ptr)
|
||||
{
|
||||
auto type = toNativeType<T>(builder);
|
||||
builder.CreateStore(llvm::ConstantInt::get(type, -1), value_ptr);
|
||||
}
|
||||
|
||||
static llvm::Value* compileUpdate(llvm::IRBuilderBase & builder, llvm::Value * lhs, llvm::Value * rhs)
|
||||
{
|
||||
return builder.CreateAnd(lhs, rhs);
|
||||
}
|
||||
|
||||
#endif
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
@ -37,6 +75,21 @@ struct AggregateFunctionGroupBitXorData
|
||||
T value = 0;
|
||||
static const char * name() { return "groupBitXor"; }
|
||||
void update(T x) { value ^= x; }
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
||||
static void compileCreate(llvm::IRBuilderBase & builder, llvm::Value * value_ptr)
|
||||
{
|
||||
auto type = toNativeType<T>(builder);
|
||||
builder.CreateStore(llvm::Constant::getNullValue(type), value_ptr);
|
||||
}
|
||||
|
||||
static llvm::Value* compileUpdate(llvm::IRBuilderBase & builder, llvm::Value * lhs, llvm::Value * rhs)
|
||||
{
|
||||
return builder.CreateXor(lhs, rhs);
|
||||
}
|
||||
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
@ -45,7 +98,7 @@ template <typename T, typename Data>
|
||||
class AggregateFunctionBitwise final : public IAggregateFunctionDataHelper<Data, AggregateFunctionBitwise<T, Data>>
|
||||
{
|
||||
public:
|
||||
AggregateFunctionBitwise(const DataTypePtr & type)
|
||||
explicit AggregateFunctionBitwise(const DataTypePtr & type)
|
||||
: IAggregateFunctionDataHelper<Data, AggregateFunctionBitwise<T, Data>>({type}, {}) {}
|
||||
|
||||
String getName() const override { return Data::name(); }
|
||||
@ -81,6 +134,68 @@ public:
|
||||
{
|
||||
assert_cast<ColumnVector<T> &>(to).getData().push_back(this->data(place).value);
|
||||
}
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
||||
bool isCompilable() const override
|
||||
{
|
||||
auto return_type = getReturnType();
|
||||
return canBeNativeType(*return_type);
|
||||
}
|
||||
|
||||
void compileCreate(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr) const override
|
||||
{
|
||||
llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
|
||||
|
||||
auto * return_type = toNativeType(b, getReturnType());
|
||||
auto * value_ptr = b.CreatePointerCast(aggregate_data_ptr, return_type->getPointerTo());
|
||||
Data::compileCreate(builder, value_ptr);
|
||||
}
|
||||
|
||||
void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const DataTypes &, const std::vector<llvm::Value *> & argument_values) const override
|
||||
{
|
||||
llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
|
||||
|
||||
auto * return_type = toNativeType(b, getReturnType());
|
||||
|
||||
auto * value_ptr = b.CreatePointerCast(aggregate_data_ptr, return_type->getPointerTo());
|
||||
auto * value = b.CreateLoad(return_type, value_ptr);
|
||||
|
||||
const auto & argument_value = argument_values[0];
|
||||
auto * result_value = Data::compileUpdate(builder, value, argument_value);
|
||||
|
||||
b.CreateStore(result_value, value_ptr);
|
||||
}
|
||||
|
||||
void compileMerge(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_dst_ptr, llvm::Value * aggregate_data_src_ptr) const override
|
||||
{
|
||||
llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
|
||||
|
||||
auto * return_type = toNativeType(b, getReturnType());
|
||||
|
||||
auto * value_dst_ptr = b.CreatePointerCast(aggregate_data_dst_ptr, return_type->getPointerTo());
|
||||
auto * value_dst = b.CreateLoad(return_type, value_dst_ptr);
|
||||
|
||||
auto * value_src_ptr = b.CreatePointerCast(aggregate_data_src_ptr, return_type->getPointerTo());
|
||||
auto * value_src = b.CreateLoad(return_type, value_src_ptr);
|
||||
|
||||
auto * result_value = Data::compileUpdate(builder, value_dst, value_src);
|
||||
|
||||
b.CreateStore(result_value, value_dst_ptr);
|
||||
}
|
||||
|
||||
llvm::Value * compileGetResult(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr) const override
|
||||
{
|
||||
llvm::IRBuilder<> & b = static_cast<llvm::IRBuilder<> &>(builder);
|
||||
|
||||
auto * return_type = toNativeType(b, getReturnType());
|
||||
auto * value_ptr = b.CreatePointerCast(aggregate_data_ptr, return_type->getPointerTo());
|
||||
|
||||
return b.CreateLoad(return_type, value_ptr);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <AggregateFunctions/AggregateFunctionSequenceMatch.h>
|
||||
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDate32.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
|
||||
#include <common/range.h>
|
||||
|
||||
@ -101,6 +101,24 @@ struct AggregateFunctionSumData
|
||||
{
|
||||
const auto * end = ptr + count;
|
||||
|
||||
if constexpr (
|
||||
(is_integer_v<T> && !is_big_int_v<T>)
|
||||
|| (IsDecimalNumber<T> && !std::is_same_v<T, Decimal256> && !std::is_same_v<T, Decimal128>))
|
||||
{
|
||||
/// For integers we can vectorize the operation if we replace the null check using a multiplication (by 0 for null, 1 for not null)
|
||||
/// https://quick-bench.com/q/MLTnfTvwC2qZFVeWHfOBR3U7a8I
|
||||
T local_sum{};
|
||||
while (ptr < end)
|
||||
{
|
||||
T multiplier = !*null_map;
|
||||
Impl::add(local_sum, *ptr * multiplier);
|
||||
++ptr;
|
||||
++null_map;
|
||||
}
|
||||
Impl::add(sum, local_sum);
|
||||
return;
|
||||
}
|
||||
|
||||
if constexpr (std::is_floating_point_v<T>)
|
||||
{
|
||||
constexpr size_t unroll_count = 128 / sizeof(T);
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include <AggregateFunctions/FactoryHelpers.h>
|
||||
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDate32.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/DataTypeUUID.h>
|
||||
@ -49,6 +50,8 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
|
||||
return res;
|
||||
else if (which.isDate())
|
||||
return std::make_shared<AggregateFunctionUniq<DataTypeDate::FieldType, Data>>(argument_types);
|
||||
else if (which.isDate32())
|
||||
return std::make_shared<AggregateFunctionUniq<DataTypeDate32::FieldType, Data>>(argument_types);
|
||||
else if (which.isDateTime())
|
||||
return std::make_shared<AggregateFunctionUniq<DataTypeDateTime::FieldType, Data>>(argument_types);
|
||||
else if (which.isStringOrFixedString())
|
||||
@ -95,6 +98,8 @@ AggregateFunctionPtr createAggregateFunctionUniq(const std::string & name, const
|
||||
return res;
|
||||
else if (which.isDate())
|
||||
return std::make_shared<AggregateFunctionUniq<DataTypeDate::FieldType, Data<DataTypeDate::FieldType>>>(argument_types);
|
||||
else if (which.isDate32())
|
||||
return std::make_shared<AggregateFunctionUniq<DataTypeDate32::FieldType, Data<DataTypeDate32::FieldType>>>(argument_types);
|
||||
else if (which.isDateTime())
|
||||
return std::make_shared<AggregateFunctionUniq<DataTypeDateTime::FieldType, Data<DataTypeDateTime::FieldType>>>(argument_types);
|
||||
else if (which.isStringOrFixedString())
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include <Common/FieldVisitorConvertToNumber.h>
|
||||
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDate32.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
|
||||
#include <functional>
|
||||
@ -51,6 +52,8 @@ namespace
|
||||
return res;
|
||||
else if (which.isDate())
|
||||
return std::make_shared<typename WithK<K, HashValueType>::template AggregateFunction<DataTypeDate::FieldType>>(argument_types, params);
|
||||
else if (which.isDate32())
|
||||
return std::make_shared<typename WithK<K, HashValueType>::template AggregateFunction<DataTypeDate32::FieldType>>(argument_types, params);
|
||||
else if (which.isDateTime())
|
||||
return std::make_shared<typename WithK<K, HashValueType>::template AggregateFunction<DataTypeDateTime::FieldType>>(argument_types, params);
|
||||
else if (which.isStringOrFixedString())
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <AggregateFunctions/AggregateFunctionUniqUpTo.h>
|
||||
#include <Common/FieldVisitorConvertToNumber.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDate32.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypeFixedString.h>
|
||||
@ -61,6 +62,8 @@ AggregateFunctionPtr createAggregateFunctionUniqUpTo(const std::string & name, c
|
||||
return res;
|
||||
else if (which.isDate())
|
||||
return std::make_shared<AggregateFunctionUniqUpTo<DataTypeDate::FieldType>>(threshold, argument_types, params);
|
||||
else if (which.isDate32())
|
||||
return std::make_shared<AggregateFunctionUniqUpTo<DataTypeDate32::FieldType>>(threshold, argument_types, params);
|
||||
else if (which.isDateTime())
|
||||
return std::make_shared<AggregateFunctionUniqUpTo<DataTypeDateTime::FieldType>>(threshold, argument_types, params);
|
||||
else if (which.isStringOrFixedString())
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
#include <Core/Settings.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDate32.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
|
||||
#include <common/range.h>
|
||||
|
||||
@ -298,11 +298,19 @@ void ConfigProcessor::doIncludesRecursive(
|
||||
{
|
||||
const auto * subst = attributes->getNamedItem(attr_name);
|
||||
attr_nodes[attr_name] = subst;
|
||||
substs_count += static_cast<size_t>(subst == nullptr);
|
||||
substs_count += static_cast<size_t>(subst != nullptr);
|
||||
}
|
||||
|
||||
if (substs_count < SUBSTITUTION_ATTRS.size() - 1) /// only one substitution is allowed
|
||||
throw Poco::Exception("several substitutions attributes set for element <" + node->nodeName() + ">");
|
||||
if (substs_count > 1) /// only one substitution is allowed
|
||||
throw Poco::Exception("More than one substitution attribute is set for element <" + node->nodeName() + ">");
|
||||
|
||||
if (node->nodeName() == "include")
|
||||
{
|
||||
if (node->hasChildNodes())
|
||||
throw Poco::Exception("<include> element must have no children");
|
||||
if (substs_count == 0)
|
||||
throw Poco::Exception("No substitution attributes set for element <include>, must have exactly one");
|
||||
}
|
||||
|
||||
/// Replace the original contents, not add to it.
|
||||
bool replace = attributes->getNamedItem("replace");
|
||||
@ -320,37 +328,57 @@ void ConfigProcessor::doIncludesRecursive(
|
||||
else if (throw_on_bad_incl)
|
||||
throw Poco::Exception(error_msg + name);
|
||||
else
|
||||
{
|
||||
if (node->nodeName() == "include")
|
||||
node->parentNode()->removeChild(node);
|
||||
|
||||
LOG_WARNING(log, "{}{}", error_msg, name);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Element & element = dynamic_cast<Element &>(*node);
|
||||
|
||||
for (const auto & attr_name : SUBSTITUTION_ATTRS)
|
||||
element.removeAttribute(attr_name);
|
||||
|
||||
if (replace)
|
||||
/// Replace the whole node not just contents.
|
||||
if (node->nodeName() == "include")
|
||||
{
|
||||
while (Node * child = node->firstChild())
|
||||
node->removeChild(child);
|
||||
const NodeListPtr children = node_to_include->childNodes();
|
||||
for (size_t i = 0, size = children->length(); i < size; ++i)
|
||||
{
|
||||
NodePtr new_node = config->importNode(children->item(i), true);
|
||||
node->parentNode()->insertBefore(new_node, node);
|
||||
}
|
||||
|
||||
element.removeAttribute("replace");
|
||||
node->parentNode()->removeChild(node);
|
||||
}
|
||||
|
||||
const NodeListPtr children = node_to_include->childNodes();
|
||||
for (size_t i = 0, size = children->length(); i < size; ++i)
|
||||
else
|
||||
{
|
||||
NodePtr new_node = config->importNode(children->item(i), true);
|
||||
node->appendChild(new_node);
|
||||
}
|
||||
Element & element = dynamic_cast<Element &>(*node);
|
||||
|
||||
const NamedNodeMapPtr from_attrs = node_to_include->attributes();
|
||||
for (size_t i = 0, size = from_attrs->length(); i < size; ++i)
|
||||
{
|
||||
element.setAttributeNode(dynamic_cast<Attr *>(config->importNode(from_attrs->item(i), true)));
|
||||
}
|
||||
for (const auto & attr_name : SUBSTITUTION_ATTRS)
|
||||
element.removeAttribute(attr_name);
|
||||
|
||||
included_something = true;
|
||||
if (replace)
|
||||
{
|
||||
while (Node * child = node->firstChild())
|
||||
node->removeChild(child);
|
||||
|
||||
element.removeAttribute("replace");
|
||||
}
|
||||
|
||||
const NodeListPtr children = node_to_include->childNodes();
|
||||
for (size_t i = 0, size = children->length(); i < size; ++i)
|
||||
{
|
||||
NodePtr new_node = config->importNode(children->item(i), true);
|
||||
node->appendChild(new_node);
|
||||
}
|
||||
|
||||
const NamedNodeMapPtr from_attrs = node_to_include->attributes();
|
||||
for (size_t i = 0, size = from_attrs->length(); i < size; ++i)
|
||||
{
|
||||
element.setAttributeNode(dynamic_cast<Attr *>(config->importNode(from_attrs->item(i), true)));
|
||||
}
|
||||
|
||||
included_something = true;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -10,16 +10,10 @@ namespace fs = std::filesystem;
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Checks if file exists without throwing an exception but with message in console.
|
||||
bool safeFsExists(const auto & path)
|
||||
bool safeFsExists(const String & path)
|
||||
{
|
||||
std::error_code ec;
|
||||
bool res = fs::exists(path, ec);
|
||||
if (ec)
|
||||
{
|
||||
std::cerr << "Can't check '" << path << "': [" << ec.value() << "] " << ec.message() << std::endl;
|
||||
}
|
||||
return res;
|
||||
return fs::exists(path, ec);
|
||||
};
|
||||
|
||||
bool configReadClient(Poco::Util::LayeredConfiguration & config, const std::string & home_path)
|
||||
|
||||
@ -237,7 +237,12 @@ public:
|
||||
// 1. Always memcpy 8 times bytes
|
||||
// 2. Use switch case extension to generate fast dispatching table
|
||||
// 3. Funcs are named callables that can be force_inlined
|
||||
//
|
||||
// NOTE: It relies on Little Endianness
|
||||
//
|
||||
// NOTE: It requires padded to 8 bytes keys (IOW you cannot pass
|
||||
// std::string here, but you can pass i.e. ColumnString::getDataAt()),
|
||||
// since it copies 8 bytes at a time.
|
||||
template <typename Self, typename KeyHolder, typename Func>
|
||||
static auto ALWAYS_INLINE dispatch(Self & self, KeyHolder && key_holder, Func && func)
|
||||
{
|
||||
|
||||
@ -22,10 +22,6 @@
|
||||
M(WriteBufferFromFileDescriptorWrite, "Number of writes (write/pwrite) to a file descriptor. Does not include sockets.") \
|
||||
M(WriteBufferFromFileDescriptorWriteFailed, "Number of times the write (write/pwrite) to a file descriptor have failed.") \
|
||||
M(WriteBufferFromFileDescriptorWriteBytes, "Number of bytes written to file descriptors. If the file is compressed, this will show compressed data size.") \
|
||||
M(ReadBufferAIORead, "") \
|
||||
M(ReadBufferAIOReadBytes, "") \
|
||||
M(WriteBufferAIOWrite, "") \
|
||||
M(WriteBufferAIOWriteBytes, "") \
|
||||
M(ReadCompressedBytes, "Number of bytes (the number of bytes before decompression) read from compressed sources (files, network).") \
|
||||
M(CompressedReadBufferBlocks, "Number of compressed blocks (the blocks of data that are compressed independent of each other) read from compressed sources (files, network).") \
|
||||
M(CompressedReadBufferBytes, "Number of uncompressed bytes (the number of bytes after decompression) read from compressed sources (files, network).") \
|
||||
@ -34,6 +30,10 @@
|
||||
M(UncompressedCacheWeightLost, "") \
|
||||
M(MMappedFileCacheHits, "") \
|
||||
M(MMappedFileCacheMisses, "") \
|
||||
M(AIOWrite, "Number of writes with Linux or FreeBSD AIO interface") \
|
||||
M(AIOWriteBytes, "Number of bytes written with Linux or FreeBSD AIO interface") \
|
||||
M(AIORead, "Number of reads with Linux or FreeBSD AIO interface") \
|
||||
M(AIOReadBytes, "Number of bytes read with Linux or FreeBSD AIO interface") \
|
||||
M(IOBufferAllocs, "") \
|
||||
M(IOBufferAllocBytes, "") \
|
||||
M(ArenaAllocChunks, "") \
|
||||
@ -43,8 +43,8 @@
|
||||
M(MarkCacheHits, "") \
|
||||
M(MarkCacheMisses, "") \
|
||||
M(CreatedReadBufferOrdinary, "") \
|
||||
M(CreatedReadBufferAIO, "") \
|
||||
M(CreatedReadBufferAIOFailed, "") \
|
||||
M(CreatedReadBufferDirectIO, "") \
|
||||
M(CreatedReadBufferDirectIOFailed, "") \
|
||||
M(CreatedReadBufferMMap, "") \
|
||||
M(CreatedReadBufferMMapFailed, "") \
|
||||
M(DiskReadElapsedMicroseconds, "Total time spent waiting for read syscall. This include reads from page cache.") \
|
||||
|
||||
@ -4,9 +4,6 @@
|
||||
#include <Common/UnicodeBar.h>
|
||||
#include <Databases/DatabaseMemory.h>
|
||||
|
||||
/// FIXME: progress bar in clickhouse-local needs to be cleared after query execution
|
||||
/// - same as it is now in clickhouse-client. Also there is no writeFinalProgress call
|
||||
/// in clickhouse-local.
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -45,6 +45,8 @@ struct ZooKeeperRequest : virtual Request
|
||||
/// If the request was sent and we didn't get the response and the error happens, then we cannot be sure was it processed or not.
|
||||
bool probably_sent = false;
|
||||
|
||||
bool restored_from_zookeeper_log = false;
|
||||
|
||||
ZooKeeperRequest() = default;
|
||||
ZooKeeperRequest(const ZooKeeperRequest &) = default;
|
||||
virtual ~ZooKeeperRequest() override = default;
|
||||
@ -172,6 +174,9 @@ struct ZooKeeperCloseResponse final : ZooKeeperResponse
|
||||
|
||||
struct ZooKeeperCreateRequest final : public CreateRequest, ZooKeeperRequest
|
||||
{
|
||||
/// used only during restore from zookeeper log
|
||||
int32_t parent_cversion = -1;
|
||||
|
||||
ZooKeeperCreateRequest() = default;
|
||||
explicit ZooKeeperCreateRequest(const CreateRequest & base) : CreateRequest(base) {}
|
||||
|
||||
@ -183,9 +188,6 @@ struct ZooKeeperCreateRequest final : public CreateRequest, ZooKeeperRequest
|
||||
bool isReadRequest() const override { return false; }
|
||||
|
||||
size_t bytesSize() const override { return CreateRequest::bytesSize() + sizeof(xid) + sizeof(has_watch); }
|
||||
|
||||
/// During recovery from log we don't rehash ACLs
|
||||
bool need_to_hash_acls = true;
|
||||
};
|
||||
|
||||
struct ZooKeeperCreateResponse final : CreateResponse, ZooKeeperResponse
|
||||
@ -362,8 +364,6 @@ struct ZooKeeperSetACLRequest final : SetACLRequest, ZooKeeperRequest
|
||||
bool isReadRequest() const override { return false; }
|
||||
|
||||
size_t bytesSize() const override { return SetACLRequest::bytesSize() + sizeof(xid); }
|
||||
|
||||
bool need_to_hash_acls = true;
|
||||
};
|
||||
|
||||
struct ZooKeeperSetACLResponse final : SetACLResponse, ZooKeeperResponse
|
||||
|
||||
@ -47,13 +47,13 @@ CompressedReadBufferFromFile::CompressedReadBufferFromFile(std::unique_ptr<ReadB
|
||||
CompressedReadBufferFromFile::CompressedReadBufferFromFile(
|
||||
const std::string & path,
|
||||
size_t estimated_size,
|
||||
size_t aio_threshold,
|
||||
size_t direct_io_threshold,
|
||||
size_t mmap_threshold,
|
||||
MMappedFileCache * mmap_cache,
|
||||
size_t buf_size,
|
||||
bool allow_different_codecs_)
|
||||
: BufferWithOwnMemory<ReadBuffer>(0)
|
||||
, p_file_in(createReadBufferFromFileBase(path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size))
|
||||
, p_file_in(createReadBufferFromFileBase(path, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache, buf_size))
|
||||
, file_in(*p_file_in)
|
||||
{
|
||||
compressed_in = &file_in;
|
||||
|
||||
@ -33,7 +33,7 @@ public:
|
||||
CompressedReadBufferFromFile(std::unique_ptr<ReadBufferFromFileBase> buf, bool allow_different_codecs_ = false);
|
||||
|
||||
CompressedReadBufferFromFile(
|
||||
const std::string & path, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache,
|
||||
const std::string & path, size_t estimated_size, size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache,
|
||||
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, bool allow_different_codecs_ = false);
|
||||
|
||||
void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block);
|
||||
|
||||
@ -267,13 +267,12 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
auto & session_auth_ids = storage.session_and_auth[session_id];
|
||||
|
||||
KeeperStorage::Node created_node;
|
||||
|
||||
Coordination::ACLs node_acls;
|
||||
if (!fixupACL(request.acls, session_auth_ids, node_acls, request.need_to_hash_acls))
|
||||
if (!fixupACL(request.acls, session_auth_ids, node_acls, !request.restored_from_zookeeper_log))
|
||||
{
|
||||
response.error = Coordination::Error::ZINVALIDACL;
|
||||
return {response_ptr, {}};
|
||||
@ -307,16 +306,28 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest
|
||||
path_created += seq_num_str.str();
|
||||
}
|
||||
|
||||
int32_t parent_cversion = request.parent_cversion;
|
||||
auto child_path = getBaseName(path_created);
|
||||
int64_t prev_parent_zxid;
|
||||
container.updateValue(parent_path, [child_path, zxid, &prev_parent_zxid] (KeeperStorage::Node & parent)
|
||||
int32_t prev_parent_cversion;
|
||||
container.updateValue(parent_path, [child_path, zxid, &prev_parent_zxid,
|
||||
parent_cversion, &prev_parent_cversion] (KeeperStorage::Node & parent)
|
||||
{
|
||||
|
||||
parent.children.insert(child_path);
|
||||
prev_parent_cversion = parent.stat.cversion;
|
||||
prev_parent_zxid = parent.stat.pzxid;
|
||||
|
||||
/// Increment sequential number even if node is not sequential
|
||||
++parent.seq_num;
|
||||
parent.children.insert(child_path);
|
||||
++parent.stat.cversion;
|
||||
prev_parent_zxid = parent.stat.pzxid;
|
||||
parent.stat.pzxid = zxid;
|
||||
|
||||
if (parent_cversion == -1)
|
||||
++parent.stat.cversion;
|
||||
else if (parent_cversion > parent.stat.cversion)
|
||||
parent.stat.cversion = parent_cversion;
|
||||
|
||||
if (zxid > parent.stat.pzxid)
|
||||
parent.stat.pzxid = zxid;
|
||||
++parent.stat.numChildren;
|
||||
});
|
||||
|
||||
@ -326,7 +337,7 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest
|
||||
if (request.is_ephemeral)
|
||||
ephemerals[session_id].emplace(path_created);
|
||||
|
||||
undo = [&storage, prev_parent_zxid, session_id, path_created, is_ephemeral = request.is_ephemeral, parent_path, child_path, acl_id]
|
||||
undo = [&storage, prev_parent_zxid, prev_parent_cversion, session_id, path_created, is_ephemeral = request.is_ephemeral, parent_path, child_path, acl_id]
|
||||
{
|
||||
storage.container.erase(path_created);
|
||||
storage.acl_map.removeUsage(acl_id);
|
||||
@ -334,11 +345,11 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest
|
||||
if (is_ephemeral)
|
||||
storage.ephemerals[session_id].erase(path_created);
|
||||
|
||||
storage.container.updateValue(parent_path, [child_path, prev_parent_zxid] (KeeperStorage::Node & undo_parent)
|
||||
storage.container.updateValue(parent_path, [child_path, prev_parent_zxid, prev_parent_cversion] (KeeperStorage::Node & undo_parent)
|
||||
{
|
||||
--undo_parent.stat.cversion;
|
||||
--undo_parent.stat.numChildren;
|
||||
--undo_parent.seq_num;
|
||||
undo_parent.stat.cversion = prev_parent_cversion;
|
||||
undo_parent.stat.pzxid = prev_parent_zxid;
|
||||
undo_parent.children.erase(child_path);
|
||||
});
|
||||
@ -394,6 +405,24 @@ struct KeeperStorageGetRequest final : public KeeperStorageRequest
|
||||
}
|
||||
};
|
||||
|
||||
namespace
|
||||
{
|
||||
/// Garbage required to apply log to "fuzzy" zookeeper snapshot
|
||||
void updateParentPzxid(const std::string & child_path, int64_t zxid, KeeperStorage::Container & container)
|
||||
{
|
||||
auto parent_path = parentPath(child_path);
|
||||
auto parent_it = container.find(parent_path);
|
||||
if (parent_it != container.end())
|
||||
{
|
||||
container.updateValue(parent_path, [zxid](KeeperStorage::Node & parent)
|
||||
{
|
||||
if (parent.stat.pzxid < zxid)
|
||||
parent.stat.pzxid = zxid;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct KeeperStorageRemoveRequest final : public KeeperStorageRequest
|
||||
{
|
||||
bool checkAuth(KeeperStorage & storage, int64_t session_id) const override
|
||||
@ -412,7 +441,7 @@ struct KeeperStorageRemoveRequest final : public KeeperStorageRequest
|
||||
}
|
||||
|
||||
using KeeperStorageRequest::KeeperStorageRequest;
|
||||
std::pair<Coordination::ZooKeeperResponsePtr, Undo> process(KeeperStorage & storage, int64_t /*zxid*/, int64_t /*session_id*/) const override
|
||||
std::pair<Coordination::ZooKeeperResponsePtr, Undo> process(KeeperStorage & storage, int64_t zxid, int64_t /*session_id*/) const override
|
||||
{
|
||||
auto & container = storage.container;
|
||||
auto & ephemerals = storage.ephemerals;
|
||||
@ -425,6 +454,8 @@ struct KeeperStorageRemoveRequest final : public KeeperStorageRequest
|
||||
auto it = container.find(request.path);
|
||||
if (it == container.end())
|
||||
{
|
||||
if (request.restored_from_zookeeper_log)
|
||||
updateParentPzxid(request.path, zxid, container);
|
||||
response.error = Coordination::Error::ZNONODE;
|
||||
}
|
||||
else if (request.version != -1 && request.version != it->value.stat.version)
|
||||
@ -437,6 +468,9 @@ struct KeeperStorageRemoveRequest final : public KeeperStorageRequest
|
||||
}
|
||||
else
|
||||
{
|
||||
if (request.restored_from_zookeeper_log)
|
||||
updateParentPzxid(request.path, zxid, container);
|
||||
|
||||
auto prev_node = it->value;
|
||||
if (prev_node.stat.ephemeralOwner != 0)
|
||||
{
|
||||
@ -719,7 +753,7 @@ struct KeeperStorageSetACLRequest final : public KeeperStorageRequest
|
||||
auto & session_auth_ids = storage.session_and_auth[session_id];
|
||||
Coordination::ACLs node_acls;
|
||||
|
||||
if (!fixupACL(request.acls, session_auth_ids, node_acls, request.need_to_hash_acls))
|
||||
if (!fixupACL(request.acls, session_auth_ids, node_acls, !request.restored_from_zookeeper_log))
|
||||
{
|
||||
response.error = Coordination::Error::ZINVALIDACL;
|
||||
return {response_ptr, {}};
|
||||
|
||||
@ -174,7 +174,22 @@ void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::st
|
||||
|
||||
LOG_INFO(log, "Deserializing data from snapshot");
|
||||
int64_t zxid_from_nodes = deserializeStorageData(storage, reader, log);
|
||||
storage.zxid = std::max(zxid, zxid_from_nodes);
|
||||
/// In ZooKeeper Snapshots can contain inconsistent state of storage. They call
|
||||
/// this inconsistent state "fuzzy". So it's guaranteed that snapshot contain all
|
||||
/// records up to zxid from snapshot name and also some records for future.
|
||||
/// But it doesn't mean that we have just some state of storage from future (like zxid + 100 log records).
|
||||
/// We have incorrect state of storage where some random log entries from future were applied....
|
||||
///
|
||||
/// In ZooKeeper they say that their transactions log is idempotent and can be applied to "fuzzy" state as is.
|
||||
/// It's true but there is no any general invariant which produces this property. They just have ad-hoc "if's" which detects
|
||||
/// "fuzzy" state inconsistencies and apply log records in special way. Several examples:
|
||||
/// https://github.com/apache/zookeeper/blob/master/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java#L453-L463
|
||||
/// https://github.com/apache/zookeeper/blob/master/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java#L476-L480
|
||||
/// https://github.com/apache/zookeeper/blob/master/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java#L547-L549
|
||||
if (zxid_from_nodes > zxid)
|
||||
LOG_WARNING(log, "ZooKeeper snapshot was in inconsistent (fuzzy) state. Will try to apply log. ZooKeeper create non fuzzy snapshot with restart. You can just restart ZooKeeper server and get consistent version.");
|
||||
|
||||
storage.zxid = zxid;
|
||||
|
||||
LOG_INFO(log, "Finished, snapshot ZXID {}", storage.zxid);
|
||||
}
|
||||
@ -210,16 +225,18 @@ void deserializeLogMagic(ReadBuffer & in)
|
||||
|
||||
static constexpr int32_t LOG_HEADER = 1514884167; /// "ZKLG"
|
||||
if (magic_header != LOG_HEADER)
|
||||
throw Exception(ErrorCodes::CORRUPTED_DATA ,"Incorrect magic header in file, expected {}, got {}", LOG_HEADER, magic_header);
|
||||
throw Exception(ErrorCodes::CORRUPTED_DATA, "Incorrect magic header in file, expected {}, got {}", LOG_HEADER, magic_header);
|
||||
|
||||
if (version != 2)
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED,"Cannot deserialize ZooKeeper data other than version 2, got version {}", version);
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot deserialize ZooKeeper data other than version 2, got version {}", version);
|
||||
}
|
||||
|
||||
|
||||
/// For some reason zookeeper stores slightly different records in log then
|
||||
/// requests. For example:
|
||||
/// class CreateTxn {
|
||||
/// ZooKeeper transactions log differs from requests. The main reason: to store records in log
|
||||
/// in some "finalized" state (for example with concrete versions).
|
||||
///
|
||||
/// Example:
|
||||
/// class CreateTxn {
|
||||
/// ustring path;
|
||||
/// buffer data;
|
||||
/// vector<org.apache.zookeeper.data.ACL> acl;
|
||||
@ -289,10 +306,9 @@ Coordination::ZooKeeperRequestPtr deserializeCreateTxn(ReadBuffer & in)
|
||||
Coordination::read(result->data, in);
|
||||
Coordination::read(result->acls, in);
|
||||
Coordination::read(result->is_ephemeral, in);
|
||||
result->need_to_hash_acls = false;
|
||||
/// How we should use it? It should just increment on request execution
|
||||
int32_t parent_c_version;
|
||||
Coordination::read(parent_c_version, in);
|
||||
Coordination::read(result->parent_cversion, in);
|
||||
|
||||
result->restored_from_zookeeper_log = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -300,6 +316,7 @@ Coordination::ZooKeeperRequestPtr deserializeDeleteTxn(ReadBuffer & in)
|
||||
{
|
||||
std::shared_ptr<Coordination::ZooKeeperRemoveRequest> result = std::make_shared<Coordination::ZooKeeperRemoveRequest>();
|
||||
Coordination::read(result->path, in);
|
||||
result->restored_from_zookeeper_log = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -309,6 +326,7 @@ Coordination::ZooKeeperRequestPtr deserializeSetTxn(ReadBuffer & in)
|
||||
Coordination::read(result->path, in);
|
||||
Coordination::read(result->data, in);
|
||||
Coordination::read(result->version, in);
|
||||
result->restored_from_zookeeper_log = true;
|
||||
/// It stores version + 1 (which should be, not for request)
|
||||
result->version -= 1;
|
||||
|
||||
@ -320,6 +338,7 @@ Coordination::ZooKeeperRequestPtr deserializeCheckVersionTxn(ReadBuffer & in)
|
||||
std::shared_ptr<Coordination::ZooKeeperCheckRequest> result = std::make_shared<Coordination::ZooKeeperCheckRequest>();
|
||||
Coordination::read(result->path, in);
|
||||
Coordination::read(result->version, in);
|
||||
result->restored_from_zookeeper_log = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -329,14 +348,19 @@ Coordination::ZooKeeperRequestPtr deserializeCreateSession(ReadBuffer & in)
|
||||
int32_t timeout;
|
||||
Coordination::read(timeout, in);
|
||||
result->session_timeout_ms = timeout;
|
||||
result->restored_from_zookeeper_log = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
Coordination::ZooKeeperRequestPtr deserializeCloseSession(ReadBuffer & in)
|
||||
Coordination::ZooKeeperRequestPtr deserializeCloseSession(ReadBuffer & in, bool empty)
|
||||
{
|
||||
std::shared_ptr<Coordination::ZooKeeperCloseRequest> result = std::make_shared<Coordination::ZooKeeperCloseRequest>();
|
||||
std::vector<std::string> data;
|
||||
Coordination::read(data, in);
|
||||
if (!empty)
|
||||
{
|
||||
std::vector<std::string> data;
|
||||
Coordination::read(data, in);
|
||||
}
|
||||
result->restored_from_zookeeper_log = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -356,14 +380,14 @@ Coordination::ZooKeeperRequestPtr deserializeSetACLTxn(ReadBuffer & in)
|
||||
Coordination::read(result->version, in);
|
||||
/// It stores version + 1 (which should be, not for request)
|
||||
result->version -= 1;
|
||||
result->need_to_hash_acls = false;
|
||||
result->restored_from_zookeeper_log = true;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
Coordination::ZooKeeperRequestPtr deserializeMultiTxn(ReadBuffer & in);
|
||||
|
||||
Coordination::ZooKeeperRequestPtr deserializeTxnImpl(ReadBuffer & in, bool subtxn)
|
||||
Coordination::ZooKeeperRequestPtr deserializeTxnImpl(ReadBuffer & in, bool subtxn, int64_t txn_length = 0)
|
||||
{
|
||||
int32_t type;
|
||||
Coordination::read(type, in);
|
||||
@ -372,6 +396,11 @@ Coordination::ZooKeeperRequestPtr deserializeTxnImpl(ReadBuffer & in, bool subtx
|
||||
if (subtxn)
|
||||
Coordination::read(sub_txn_length, in);
|
||||
|
||||
bool empty_txn = !subtxn && txn_length == 32; /// Possible for old-style CloseTxn's
|
||||
|
||||
if (empty_txn && type != -11)
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Empty non close-session transaction found");
|
||||
|
||||
int64_t in_count_before = in.count();
|
||||
|
||||
switch (type)
|
||||
@ -398,7 +427,7 @@ Coordination::ZooKeeperRequestPtr deserializeTxnImpl(ReadBuffer & in, bool subtx
|
||||
result = deserializeCreateSession(in);
|
||||
break;
|
||||
case -11:
|
||||
result = deserializeCloseSession(in);
|
||||
result = deserializeCloseSession(in, empty_txn);
|
||||
break;
|
||||
case -1:
|
||||
result = deserializeErrorTxn(in);
|
||||
@ -442,7 +471,7 @@ bool hasErrorsInMultiRequest(Coordination::ZooKeeperRequestPtr request)
|
||||
if (request == nullptr)
|
||||
return true;
|
||||
|
||||
for (const auto & subrequest : dynamic_cast<Coordination::ZooKeeperMultiRequest *>(request.get())->requests) //-V522
|
||||
for (const auto & subrequest : dynamic_cast<Coordination::ZooKeeperMultiRequest *>(request.get())->requests) // -V522
|
||||
if (subrequest == nullptr)
|
||||
return true;
|
||||
return false;
|
||||
@ -470,7 +499,7 @@ bool deserializeTxn(KeeperStorage & storage, ReadBuffer & in, Poco::Logger * /*l
|
||||
int64_t time;
|
||||
Coordination::read(time, in);
|
||||
|
||||
Coordination::ZooKeeperRequestPtr request = deserializeTxnImpl(in, false);
|
||||
Coordination::ZooKeeperRequestPtr request = deserializeTxnImpl(in, false, txn_len);
|
||||
|
||||
/// Skip all other bytes
|
||||
int64_t bytes_read = in.count() - count_before;
|
||||
|
||||
@ -62,6 +62,8 @@ void ExternalResultDescription::init(const Block & sample_block_)
|
||||
types.emplace_back(ValueType::vtString, is_nullable);
|
||||
else if (which.isDate())
|
||||
types.emplace_back(ValueType::vtDate, is_nullable);
|
||||
else if (which.isDate32())
|
||||
types.emplace_back(ValueType::vtDate32, is_nullable);
|
||||
else if (which.isDateTime())
|
||||
types.emplace_back(ValueType::vtDateTime, is_nullable);
|
||||
else if (which.isUUID())
|
||||
|
||||
@ -26,6 +26,7 @@ struct ExternalResultDescription
|
||||
vtEnum16,
|
||||
vtString,
|
||||
vtDate,
|
||||
vtDate32,
|
||||
vtDateTime,
|
||||
vtUUID,
|
||||
vtDateTime64,
|
||||
|
||||
@ -89,6 +89,9 @@ void insertPostgreSQLValue(
|
||||
case ExternalResultDescription::ValueType::vtDate:
|
||||
assert_cast<ColumnUInt16 &>(column).insertValue(UInt16{LocalDate{std::string(value)}.getDayNum()});
|
||||
break;
|
||||
case ExternalResultDescription::ValueType::vtDate32:
|
||||
assert_cast<ColumnInt32 &>(column).insertValue(Int32{LocalDate{std::string(value)}.getExtenedDayNum()});
|
||||
break;
|
||||
case ExternalResultDescription::ValueType::vtDateTime:
|
||||
{
|
||||
ReadBufferFromString in(value);
|
||||
|
||||
@ -108,7 +108,7 @@ class IColumn;
|
||||
M(Bool, compile_expressions, true, "Compile some scalar functions and operators to native code.", 0) \
|
||||
M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \
|
||||
M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \
|
||||
M(UInt64, min_count_to_compile_aggregate_expression, 3, "The number of identical aggregate expressions before they are JIT-compiled", 0) \
|
||||
M(UInt64, min_count_to_compile_aggregate_expression, 0, "The number of identical aggregate expressions before they are JIT-compiled", 0) \
|
||||
M(UInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.", 0) \
|
||||
M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \
|
||||
M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \
|
||||
|
||||
@ -39,6 +39,7 @@ enum class TypeIndex
|
||||
Float32,
|
||||
Float64,
|
||||
Date,
|
||||
Date32,
|
||||
DateTime,
|
||||
DateTime64,
|
||||
String,
|
||||
@ -257,6 +258,7 @@ inline constexpr const char * getTypeName(TypeIndex idx)
|
||||
case TypeIndex::Float32: return "Float32";
|
||||
case TypeIndex::Float64: return "Float64";
|
||||
case TypeIndex::Date: return "Date";
|
||||
case TypeIndex::Date32: return "Date32";
|
||||
case TypeIndex::DateTime: return "DateTime";
|
||||
case TypeIndex::DateTime64: return "DateTime64";
|
||||
case TypeIndex::String: return "String";
|
||||
|
||||
@ -192,6 +192,7 @@ bool callOnIndexAndDataType(TypeIndex number, F && f, ExtraArgs && ... args)
|
||||
case TypeIndex::Decimal256: return f(TypePair<DataTypeDecimal<Decimal256>, T>(), std::forward<ExtraArgs>(args)...);
|
||||
|
||||
case TypeIndex::Date: return f(TypePair<DataTypeDate, T>(), std::forward<ExtraArgs>(args)...);
|
||||
case TypeIndex::Date32: return f(TypePair<DataTypeDate, T>(), std::forward<ExtraArgs>(args)...);
|
||||
case TypeIndex::DateTime: return f(TypePair<DataTypeDateTime, T>(), std::forward<ExtraArgs>(args)...);
|
||||
case TypeIndex::DateTime64: return f(TypePair<DataTypeDateTime64, T>(), std::forward<ExtraArgs>(args)...);
|
||||
|
||||
|
||||
23
src/DataTypes/DataTypeDate32.cpp
Normal file
23
src/DataTypes/DataTypeDate32.cpp
Normal file
@ -0,0 +1,23 @@
|
||||
#include <DataTypes/DataTypeDate32.h>
|
||||
#include <DataTypes/DataTypeFactory.h>
|
||||
#include <DataTypes/Serializations/SerializationDate32.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
bool DataTypeDate32::equals(const IDataType & rhs) const
|
||||
{
|
||||
return typeid(rhs) == typeid(*this);
|
||||
}
|
||||
|
||||
SerializationPtr DataTypeDate32::doGetDefaultSerialization() const
|
||||
{
|
||||
return std::make_shared<SerializationDate32>();
|
||||
}
|
||||
|
||||
void registerDataTypeDate32(DataTypeFactory & factory)
|
||||
{
|
||||
factory.registerSimpleDataType(
|
||||
"Date32", [] { return DataTypePtr(std::make_shared<DataTypeDate32>()); }, DataTypeFactory::CaseInsensitive);
|
||||
}
|
||||
|
||||
}
|
||||
23
src/DataTypes/DataTypeDate32.h
Normal file
23
src/DataTypes/DataTypeDate32.h
Normal file
@ -0,0 +1,23 @@
|
||||
#pragma once
|
||||
|
||||
#include <DataTypes/DataTypeNumberBase.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
class DataTypeDate32 final : public DataTypeNumberBase<Int32>
|
||||
{
|
||||
public:
|
||||
static constexpr auto family_name = "Date32";
|
||||
|
||||
TypeIndex getTypeId() const override { return TypeIndex::Date32; }
|
||||
const char * getFamilyName() const override { return family_name; }
|
||||
|
||||
bool canBeUsedAsVersion() const override { return true; }
|
||||
bool canBeInsideNullable() const override { return true; }
|
||||
|
||||
bool equals(const IDataType & rhs) const override;
|
||||
|
||||
protected:
|
||||
SerializationPtr doGetDefaultSerialization() const override;
|
||||
};
|
||||
}
|
||||
@ -194,6 +194,7 @@ DataTypeFactory::DataTypeFactory()
|
||||
registerDataTypeNumbers(*this);
|
||||
registerDataTypeDecimal(*this);
|
||||
registerDataTypeDate(*this);
|
||||
registerDataTypeDate32(*this);
|
||||
registerDataTypeDateTime(*this);
|
||||
registerDataTypeString(*this);
|
||||
registerDataTypeFixedString(*this);
|
||||
|
||||
@ -69,6 +69,7 @@ private:
|
||||
void registerDataTypeNumbers(DataTypeFactory & factory);
|
||||
void registerDataTypeDecimal(DataTypeFactory & factory);
|
||||
void registerDataTypeDate(DataTypeFactory & factory);
|
||||
void registerDataTypeDate32(DataTypeFactory & factory);
|
||||
void registerDataTypeDateTime(DataTypeFactory & factory);
|
||||
void registerDataTypeString(DataTypeFactory & factory);
|
||||
void registerDataTypeFixedString(DataTypeFactory & factory);
|
||||
|
||||
@ -78,6 +78,8 @@ MutableColumnUniquePtr DataTypeLowCardinality::createColumnUniqueImpl(const IDat
|
||||
return creator(static_cast<ColumnFixedString *>(nullptr));
|
||||
else if (which.isDate())
|
||||
return creator(static_cast<ColumnVector<UInt16> *>(nullptr));
|
||||
else if (which.isDate32())
|
||||
return creator(static_cast<ColumnVector<Int32> *>(nullptr));
|
||||
else if (which.isDateTime())
|
||||
return creator(static_cast<ColumnVector<UInt32> *>(nullptr));
|
||||
else if (which.isUUID())
|
||||
|
||||
@ -1,11 +1,13 @@
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Columns/ColumnMap.h>
|
||||
#include <Columns/ColumnLowCardinality.h>
|
||||
|
||||
#include <DataTypes/DataTypeLowCardinality.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/DataTypeMap.h>
|
||||
|
||||
#include <Common/assert_cast.h>
|
||||
|
||||
@ -39,6 +41,11 @@ DataTypePtr recursiveRemoveLowCardinality(const DataTypePtr & type)
|
||||
return std::make_shared<DataTypeTuple>(elements);
|
||||
}
|
||||
|
||||
if (const auto * map_type = typeid_cast<const DataTypeMap *>(type.get()))
|
||||
{
|
||||
return std::make_shared<DataTypeMap>(recursiveRemoveLowCardinality(map_type->getKeyType()), recursiveRemoveLowCardinality(map_type->getValueType()));
|
||||
}
|
||||
|
||||
if (const auto * low_cardinality_type = typeid_cast<const DataTypeLowCardinality *>(type.get()))
|
||||
return low_cardinality_type->getDictionaryType();
|
||||
|
||||
@ -78,6 +85,16 @@ ColumnPtr recursiveRemoveLowCardinality(const ColumnPtr & column)
|
||||
return ColumnTuple::create(columns);
|
||||
}
|
||||
|
||||
if (const auto * column_map = typeid_cast<const ColumnMap *>(column.get()))
|
||||
{
|
||||
const auto & nested = column_map->getNestedColumnPtr();
|
||||
auto nested_no_lc = recursiveRemoveLowCardinality(nested);
|
||||
if (nested.get() == nested_no_lc.get())
|
||||
return column;
|
||||
|
||||
return ColumnMap::create(nested_no_lc);
|
||||
}
|
||||
|
||||
if (const auto * column_low_cardinality = typeid_cast<const ColumnLowCardinality *>(column.get()))
|
||||
return column_low_cardinality->convertToFullColumn();
|
||||
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
#include <DataTypes/DataTypeMap.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/DataTypeLowCardinality.h>
|
||||
#include <DataTypes/DataTypeFactory.h>
|
||||
#include <DataTypes/Serializations/SerializationMap.h>
|
||||
#include <Parsers/IAST.h>
|
||||
@ -53,12 +54,24 @@ DataTypeMap::DataTypeMap(const DataTypePtr & key_type_, const DataTypePtr & valu
|
||||
|
||||
void DataTypeMap::assertKeyType() const
|
||||
{
|
||||
if (!key_type->isValueRepresentedByInteger()
|
||||
bool type_error = false;
|
||||
if (key_type->getTypeId() == TypeIndex::LowCardinality)
|
||||
{
|
||||
const auto & low_cardinality_data_type = assert_cast<const DataTypeLowCardinality &>(*key_type);
|
||||
if (!isStringOrFixedString(*(low_cardinality_data_type.getDictionaryType())))
|
||||
type_error = true;
|
||||
}
|
||||
else if (!key_type->isValueRepresentedByInteger()
|
||||
&& !isStringOrFixedString(*key_type)
|
||||
&& !WhichDataType(key_type).isNothing()
|
||||
&& !WhichDataType(key_type).isUUID())
|
||||
{
|
||||
type_error = true;
|
||||
}
|
||||
|
||||
if (type_error)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Type of Map key must be a type, that can be represented by integer or string or UUID,"
|
||||
"Type of Map key must be a type, that can be represented by integer or String or FixedString (possibly LowCardinality) or UUID,"
|
||||
" but {} given", key_type->getName());
|
||||
}
|
||||
|
||||
|
||||
@ -322,8 +322,10 @@ struct WhichDataType
|
||||
constexpr bool isEnum() const { return isEnum8() || isEnum16(); }
|
||||
|
||||
constexpr bool isDate() const { return idx == TypeIndex::Date; }
|
||||
constexpr bool isDate32() const { return idx == TypeIndex::Date32; }
|
||||
constexpr bool isDateTime() const { return idx == TypeIndex::DateTime; }
|
||||
constexpr bool isDateTime64() const { return idx == TypeIndex::DateTime64; }
|
||||
constexpr bool isDateOrDate32() const { return isDate() || isDate32(); }
|
||||
|
||||
constexpr bool isString() const { return idx == TypeIndex::String; }
|
||||
constexpr bool isFixedString() const { return idx == TypeIndex::FixedString; }
|
||||
@ -347,6 +349,10 @@ struct WhichDataType
|
||||
template <typename T>
|
||||
inline bool isDate(const T & data_type) { return WhichDataType(data_type).isDate(); }
|
||||
template <typename T>
|
||||
inline bool isDate32(const T & data_type) { return WhichDataType(data_type).isDate32(); }
|
||||
template <typename T>
|
||||
inline bool isDateOrDate32(const T & data_type) { return WhichDataType(data_type).isDateOrDate32(); }
|
||||
template <typename T>
|
||||
inline bool isDateTime(const T & data_type) { return WhichDataType(data_type).isDateTime(); }
|
||||
template <typename T>
|
||||
inline bool isDateTime64(const T & data_type) { return WhichDataType(data_type).isDateTime64(); }
|
||||
|
||||
@ -29,7 +29,7 @@ namespace ErrorCodes
|
||||
static inline bool typeIsSigned(const IDataType & type)
|
||||
{
|
||||
WhichDataType data_type(type);
|
||||
return data_type.isNativeInt() || data_type.isFloat();
|
||||
return data_type.isNativeInt() || data_type.isFloat() || data_type.isEnum();
|
||||
}
|
||||
|
||||
static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const IDataType & type)
|
||||
@ -57,6 +57,10 @@ static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const IDa
|
||||
return builder.getFloatTy();
|
||||
else if (data_type.isFloat64())
|
||||
return builder.getDoubleTy();
|
||||
else if (data_type.isEnum8())
|
||||
return builder.getInt8Ty();
|
||||
else if (data_type.isEnum16())
|
||||
return builder.getInt16Ty();
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
@ -109,7 +113,7 @@ static inline bool canBeNativeType(const IDataType & type)
|
||||
return canBeNativeType(*data_type_nullable.getNestedType());
|
||||
}
|
||||
|
||||
return data_type.isNativeInt() || data_type.isNativeUInt() || data_type.isFloat() || data_type.isDate();
|
||||
return data_type.isNativeInt() || data_type.isNativeUInt() || data_type.isFloat() || data_type.isDate() || data_type.isEnum();
|
||||
}
|
||||
|
||||
static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const DataTypePtr & type)
|
||||
@ -266,7 +270,7 @@ static inline llvm::Constant * getColumnNativeValue(llvm::IRBuilderBase & builde
|
||||
{
|
||||
return llvm::ConstantInt::get(type, column.getUInt(index));
|
||||
}
|
||||
else if (column_data_type.isNativeInt())
|
||||
else if (column_data_type.isNativeInt() || column_data_type.isEnum())
|
||||
{
|
||||
return llvm::ConstantInt::get(type, column.getInt(index));
|
||||
}
|
||||
|
||||
78
src/DataTypes/Serializations/SerializationDate32.cpp
Normal file
78
src/DataTypes/Serializations/SerializationDate32.cpp
Normal file
@ -0,0 +1,78 @@
|
||||
#include <DataTypes/Serializations/SerializationDate32.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
|
||||
#include <Common/assert_cast.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
void SerializationDate32::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
|
||||
{
|
||||
writeDateText(ExtendedDayNum(assert_cast<const ColumnInt32 &>(column).getData()[row_num]), ostr);
|
||||
}
|
||||
|
||||
void SerializationDate32::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||
{
|
||||
deserializeTextEscaped(column, istr, settings);
|
||||
}
|
||||
|
||||
void SerializationDate32::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
|
||||
{
|
||||
ExtendedDayNum x;
|
||||
readDateText(x, istr);
|
||||
assert_cast<ColumnInt32 &>(column).getData().push_back(x);
|
||||
}
|
||||
|
||||
void SerializationDate32::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
serializeText(column, row_num, ostr, settings);
|
||||
}
|
||||
|
||||
void SerializationDate32::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
writeChar('\'', ostr);
|
||||
serializeText(column, row_num, ostr, settings);
|
||||
writeChar('\'', ostr);
|
||||
}
|
||||
|
||||
void SerializationDate32::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
|
||||
{
|
||||
ExtendedDayNum x;
|
||||
assertChar('\'', istr);
|
||||
readDateText(x, istr);
|
||||
assertChar('\'', istr);
|
||||
assert_cast<ColumnInt32 &>(column).getData().push_back(x); /// It's important to do this at the end - for exception safety.
|
||||
}
|
||||
|
||||
void SerializationDate32::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
writeChar('"', ostr);
|
||||
serializeText(column, row_num, ostr, settings);
|
||||
writeChar('"', ostr);
|
||||
}
|
||||
|
||||
void SerializationDate32::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
|
||||
{
|
||||
ExtendedDayNum x;
|
||||
assertChar('"', istr);
|
||||
readDateText(x, istr);
|
||||
assertChar('"', istr);
|
||||
assert_cast<ColumnInt32 &>(column).getData().push_back(x);
|
||||
}
|
||||
|
||||
void SerializationDate32::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
|
||||
{
|
||||
writeChar('"', ostr);
|
||||
serializeText(column, row_num, ostr, settings);
|
||||
writeChar('"', ostr);
|
||||
}
|
||||
|
||||
void SerializationDate32::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
|
||||
{
|
||||
LocalDate value;
|
||||
readCSV(value, istr);
|
||||
assert_cast<ColumnInt32 &>(column).getData().push_back(value.getExtenedDayNum());
|
||||
}
|
||||
}
|
||||
21
src/DataTypes/Serializations/SerializationDate32.h
Normal file
21
src/DataTypes/Serializations/SerializationDate32.h
Normal file
@ -0,0 +1,21 @@
|
||||
#pragma once
|
||||
|
||||
#include <DataTypes/Serializations/SerializationNumber.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
class SerializationDate32 final : public SerializationNumber<Int32>
|
||||
{
|
||||
public:
|
||||
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
|
||||
void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
|
||||
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
|
||||
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
|
||||
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
|
||||
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
|
||||
};
|
||||
}
|
||||
@ -16,6 +16,7 @@ SRCS(
|
||||
DataTypeCustomIPv4AndIPv6.cpp
|
||||
DataTypeCustomSimpleAggregateFunction.cpp
|
||||
DataTypeDate.cpp
|
||||
DataTypeDate32.cpp
|
||||
DataTypeDateTime.cpp
|
||||
DataTypeDateTime64.cpp
|
||||
DataTypeDecimalBase.cpp
|
||||
@ -45,6 +46,7 @@ SRCS(
|
||||
Serializations/SerializationArray.cpp
|
||||
Serializations/SerializationCustomSimpleText.cpp
|
||||
Serializations/SerializationDate.cpp
|
||||
Serializations/SerializationDate32.cpp
|
||||
Serializations/SerializationDateTime.cpp
|
||||
Serializations/SerializationDateTime64.cpp
|
||||
Serializations/SerializationDecimal.cpp
|
||||
|
||||
@ -26,8 +26,10 @@
|
||||
namespace ProfileEvents
|
||||
{
|
||||
extern const Event FileOpen;
|
||||
extern const Event WriteBufferAIOWrite;
|
||||
extern const Event WriteBufferAIOWriteBytes;
|
||||
extern const Event AIOWrite;
|
||||
extern const Event AIOWriteBytes;
|
||||
extern const Event AIORead;
|
||||
extern const Event AIOReadBytes;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
@ -531,8 +533,8 @@ public:
|
||||
|
||||
auto bytes_written = eventResult(event);
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::WriteBufferAIOWrite);
|
||||
ProfileEvents::increment(ProfileEvents::WriteBufferAIOWriteBytes, bytes_written);
|
||||
ProfileEvents::increment(ProfileEvents::AIOWrite);
|
||||
ProfileEvents::increment(ProfileEvents::AIOWriteBytes, bytes_written);
|
||||
|
||||
if (bytes_written != static_cast<decltype(bytes_written)>(block_size * buffer_size_in_blocks))
|
||||
throw Exception(ErrorCodes::AIO_WRITE_ERROR,
|
||||
@ -600,6 +602,9 @@ public:
|
||||
buffer_size_in_bytes,
|
||||
read_bytes);
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::AIORead);
|
||||
ProfileEvents::increment(ProfileEvents::AIOReadBytes, read_bytes);
|
||||
|
||||
SSDCacheBlock block(block_size);
|
||||
|
||||
for (size_t i = 0; i < blocks_length; ++i)
|
||||
@ -687,6 +692,9 @@ public:
|
||||
throw Exception(ErrorCodes::AIO_READ_ERROR,
|
||||
"GC: AIO failed to read file ({}). Expected bytes ({}). Actual bytes ({})", file_path, block_size, read_bytes);
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::AIORead);
|
||||
ProfileEvents::increment(ProfileEvents::AIOReadBytes, read_bytes);
|
||||
|
||||
char * request_buffer = getRequestBuffer(request);
|
||||
|
||||
// Unpoison the memory returned from an uninstrumented system function.
|
||||
|
||||
@ -90,17 +90,17 @@ DiskCacheWrapper::readFile(
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
size_t estimated_size,
|
||||
size_t aio_threshold,
|
||||
size_t direct_io_threshold,
|
||||
size_t mmap_threshold,
|
||||
MMappedFileCache * mmap_cache) const
|
||||
{
|
||||
if (!cache_file_predicate(path))
|
||||
return DiskDecorator::readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
|
||||
return DiskDecorator::readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
|
||||
|
||||
LOG_DEBUG(log, "Read file {} from cache", backQuote(path));
|
||||
|
||||
if (cache_disk->exists(path))
|
||||
return cache_disk->readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
|
||||
return cache_disk->readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
|
||||
|
||||
auto metadata = acquireDownloadMetadata(path);
|
||||
|
||||
@ -134,7 +134,7 @@ DiskCacheWrapper::readFile(
|
||||
|
||||
auto tmp_path = path + ".tmp";
|
||||
{
|
||||
auto src_buffer = DiskDecorator::readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
|
||||
auto src_buffer = DiskDecorator::readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
|
||||
auto dst_buffer = cache_disk->writeFile(tmp_path, buf_size, WriteMode::Rewrite);
|
||||
copyData(*src_buffer, *dst_buffer);
|
||||
}
|
||||
@ -158,9 +158,9 @@ DiskCacheWrapper::readFile(
|
||||
}
|
||||
|
||||
if (metadata->status == DOWNLOADED)
|
||||
return cache_disk->readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
|
||||
return cache_disk->readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
|
||||
|
||||
return DiskDecorator::readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
|
||||
return DiskDecorator::readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
|
||||
}
|
||||
|
||||
std::unique_ptr<WriteBufferFromFileBase>
|
||||
|
||||
@ -38,7 +38,7 @@ public:
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
size_t estimated_size,
|
||||
size_t aio_threshold,
|
||||
size_t direct_io_threshold,
|
||||
size_t mmap_threshold,
|
||||
MMappedFileCache * mmap_cache) const override;
|
||||
|
||||
|
||||
@ -115,9 +115,9 @@ void DiskDecorator::listFiles(const String & path, std::vector<String> & file_na
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase>
|
||||
DiskDecorator::readFile(
|
||||
const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const
|
||||
const String & path, size_t buf_size, size_t estimated_size, size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const
|
||||
{
|
||||
return delegate->readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
|
||||
return delegate->readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
|
||||
}
|
||||
|
||||
std::unique_ptr<WriteBufferFromFileBase>
|
||||
|
||||
@ -39,7 +39,7 @@ public:
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
size_t estimated_size,
|
||||
size_t aio_threshold,
|
||||
size_t direct_io_threshold,
|
||||
size_t mmap_threshold,
|
||||
MMappedFileCache * mmap_cache) const override;
|
||||
|
||||
|
||||
@ -211,9 +211,9 @@ void DiskLocal::replaceFile(const String & from_path, const String & to_path)
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase>
|
||||
DiskLocal::readFile(
|
||||
const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const
|
||||
const String & path, size_t buf_size, size_t estimated_size, size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const
|
||||
{
|
||||
return createReadBufferFromFileBase(fs::path(disk_path) / path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size);
|
||||
return createReadBufferFromFileBase(fs::path(disk_path) / path, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache, buf_size);
|
||||
}
|
||||
|
||||
std::unique_ptr<WriteBufferFromFileBase>
|
||||
|
||||
@ -74,7 +74,7 @@ public:
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
size_t estimated_size,
|
||||
size_t aio_threshold,
|
||||
size_t direct_io_threshold,
|
||||
size_t mmap_threshold,
|
||||
MMappedFileCache * mmap_cache) const override;
|
||||
|
||||
|
||||
@ -66,7 +66,7 @@ public:
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
size_t estimated_size,
|
||||
size_t aio_threshold,
|
||||
size_t direct_io_threshold,
|
||||
size_t mmap_threshold,
|
||||
MMappedFileCache * mmap_cache) const override;
|
||||
|
||||
|
||||
@ -187,11 +187,11 @@ void DiskRestartProxy::listFiles(const String & path, std::vector<String> & file
|
||||
}
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> DiskRestartProxy::readFile(
|
||||
const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache)
|
||||
const String & path, size_t buf_size, size_t estimated_size, size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache)
|
||||
const
|
||||
{
|
||||
ReadLock lock (mutex);
|
||||
auto impl = DiskDecorator::readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
|
||||
auto impl = DiskDecorator::readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache);
|
||||
return std::make_unique<RestartAwareReadBuffer>(*this, std::move(impl));
|
||||
}
|
||||
|
||||
|
||||
@ -47,7 +47,7 @@ public:
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
size_t estimated_size,
|
||||
size_t aio_threshold,
|
||||
size_t direct_io_threshold,
|
||||
size_t mmap_threshold,
|
||||
MMappedFileCache * mmap_cache) const override;
|
||||
std::unique_ptr<WriteBufferFromFileBase> writeFile(const String & path, size_t buf_size, WriteMode mode) override;
|
||||
|
||||
@ -48,7 +48,7 @@ public:
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
size_t estimated_size,
|
||||
size_t aio_threshold,
|
||||
size_t direct_io_threshold,
|
||||
size_t mmap_threshold,
|
||||
MMappedFileCache * mmap_cache) const override;
|
||||
|
||||
|
||||
@ -156,7 +156,7 @@ public:
|
||||
const String & path,
|
||||
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
|
||||
size_t estimated_size = 0,
|
||||
size_t aio_threshold = 0,
|
||||
size_t direct_io_threshold = 0,
|
||||
size_t mmap_threshold = 0,
|
||||
MMappedFileCache * mmap_cache = nullptr) const = 0;
|
||||
|
||||
|
||||
@ -417,7 +417,11 @@ void IDiskRemote::removeDirectory(const String & path)
|
||||
|
||||
DiskDirectoryIteratorPtr IDiskRemote::iterateDirectory(const String & path)
|
||||
{
|
||||
return std::make_unique<RemoteDiskDirectoryIterator>(metadata_path + path, path);
|
||||
fs::path meta_path = fs::path(metadata_path) / path;
|
||||
if (fs::exists(meta_path) && fs::is_directory(meta_path))
|
||||
return std::make_unique<RemoteDiskDirectoryIterator>(meta_path, path);
|
||||
else
|
||||
return std::make_unique<RemoteDiskDirectoryIterator>();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -193,6 +193,7 @@ struct IDiskRemote::Metadata
|
||||
class RemoteDiskDirectoryIterator final : public IDiskDirectoryIterator
|
||||
{
|
||||
public:
|
||||
RemoteDiskDirectoryIterator() {}
|
||||
RemoteDiskDirectoryIterator(const String & full_path, const String & folder_path_) : iter(full_path), folder_path(folder_path_) {}
|
||||
|
||||
void next() override { ++iter; }
|
||||
|
||||
@ -77,7 +77,7 @@ public:
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
size_t estimated_size,
|
||||
size_t aio_threshold,
|
||||
size_t direct_io_threshold,
|
||||
size_t mmap_threshold,
|
||||
MMappedFileCache * mmap_cache) const override;
|
||||
|
||||
|
||||
@ -42,6 +42,11 @@ struct ToYearWeekImpl
|
||||
YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode | static_cast<UInt32>(WeekModeFlag::YEAR));
|
||||
return yw.first * 100 + yw.second;
|
||||
}
|
||||
static inline UInt32 execute(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone)
|
||||
{
|
||||
YearWeek yw = time_zone.toYearWeek(ExtendedDayNum (d), week_mode | static_cast<UInt32>(WeekModeFlag::YEAR));
|
||||
return yw.first * 100 + yw.second;
|
||||
}
|
||||
static inline UInt32 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone)
|
||||
{
|
||||
YearWeek yw = time_zone.toYearWeek(DayNum(d), week_mode | static_cast<UInt32>(WeekModeFlag::YEAR));
|
||||
@ -65,6 +70,10 @@ struct ToStartOfWeekImpl
|
||||
return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
|
||||
// return time_zone.toFirstDayNumOfWeek(t, week_mode);
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d), week_mode);
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d), week_mode);
|
||||
@ -88,6 +97,11 @@ struct ToWeekImpl
|
||||
YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode);
|
||||
return yw.second;
|
||||
}
|
||||
static inline UInt8 execute(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone)
|
||||
{
|
||||
YearWeek yw = time_zone.toYearWeek(ExtendedDayNum(d), week_mode);
|
||||
return yw.second;
|
||||
}
|
||||
static inline UInt8 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone)
|
||||
{
|
||||
YearWeek yw = time_zone.toYearWeek(DayNum(d), week_mode);
|
||||
|
||||
@ -46,6 +46,7 @@ struct ZeroTransform
|
||||
{
|
||||
static inline UInt16 execute(Int64, const DateLUTImpl &) { return 0; }
|
||||
static inline UInt16 execute(UInt32, const DateLUTImpl &) { return 0; }
|
||||
static inline UInt16 execute(Int32, const DateLUTImpl &) { return 0; }
|
||||
static inline UInt16 execute(UInt16, const DateLUTImpl &) { return 0; }
|
||||
};
|
||||
|
||||
@ -61,6 +62,10 @@ struct ToDateImpl
|
||||
{
|
||||
return UInt16(time_zone.toDayNum(t));
|
||||
}
|
||||
static inline UInt16 execute(Int32, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl &)
|
||||
{
|
||||
return d;
|
||||
@ -82,6 +87,10 @@ struct ToStartOfDayImpl
|
||||
{
|
||||
return time_zone.toDate(t);
|
||||
}
|
||||
static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toDate(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toDate(ExtendedDayNum(d));
|
||||
@ -104,6 +113,10 @@ struct ToMondayImpl
|
||||
//return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t));
|
||||
return time_zone.toFirstDayNumOfWeek(t);
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d));
|
||||
@ -124,6 +137,10 @@ struct ToStartOfMonthImpl
|
||||
{
|
||||
return time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t));
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toFirstDayNumOfMonth(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toFirstDayNumOfMonth(ExtendedDayNum(d));
|
||||
@ -144,6 +161,10 @@ struct ToStartOfQuarterImpl
|
||||
{
|
||||
return time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t));
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toFirstDayNumOfQuarter(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toFirstDayNumOfQuarter(ExtendedDayNum(d));
|
||||
@ -164,6 +185,10 @@ struct ToStartOfYearImpl
|
||||
{
|
||||
return time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t));
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toFirstDayNumOfYear(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toFirstDayNumOfYear(ExtendedDayNum(d));
|
||||
@ -186,7 +211,10 @@ struct ToTimeImpl
|
||||
{
|
||||
return time_zone.toTime(t) + 86400;
|
||||
}
|
||||
|
||||
static inline UInt32 execute(Int32, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
}
|
||||
static inline UInt32 execute(UInt16, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
@ -207,6 +235,10 @@ struct ToStartOfMinuteImpl
|
||||
{
|
||||
return time_zone.toStartOfMinute(t);
|
||||
}
|
||||
static inline UInt32 execute(Int32, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
}
|
||||
static inline UInt32 execute(UInt16, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
@ -242,6 +274,10 @@ struct ToStartOfSecondImpl
|
||||
{
|
||||
throw Exception("Illegal type DateTime of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
static inline UInt32 execute(Int32, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
}
|
||||
static inline UInt32 execute(UInt16, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
@ -262,6 +298,10 @@ struct ToStartOfFiveMinuteImpl
|
||||
{
|
||||
return time_zone.toStartOfFiveMinute(t);
|
||||
}
|
||||
static inline UInt32 execute(Int32, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
}
|
||||
static inline UInt32 execute(UInt16, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
@ -282,6 +322,10 @@ struct ToStartOfTenMinutesImpl
|
||||
{
|
||||
return time_zone.toStartOfTenMinutes(t);
|
||||
}
|
||||
static inline UInt32 execute(Int32, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
}
|
||||
static inline UInt32 execute(UInt16, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
@ -302,6 +346,10 @@ struct ToStartOfFifteenMinutesImpl
|
||||
{
|
||||
return time_zone.toStartOfFifteenMinutes(t);
|
||||
}
|
||||
static inline UInt32 execute(Int32, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
}
|
||||
static inline UInt32 execute(UInt16, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
@ -326,6 +374,11 @@ struct TimeSlotImpl
|
||||
return t / 1800 * 1800;
|
||||
}
|
||||
|
||||
static inline UInt32 execute(Int32, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
}
|
||||
|
||||
static inline UInt32 execute(UInt16, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
@ -348,6 +401,11 @@ struct ToStartOfHourImpl
|
||||
return time_zone.toStartOfHour(t);
|
||||
}
|
||||
|
||||
static inline UInt32 execute(Int32, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
}
|
||||
|
||||
static inline UInt32 execute(UInt16, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
@ -368,6 +426,10 @@ struct ToYearImpl
|
||||
{
|
||||
return time_zone.toYear(t);
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toYear(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toYear(ExtendedDayNum(d));
|
||||
@ -388,6 +450,10 @@ struct ToQuarterImpl
|
||||
{
|
||||
return time_zone.toQuarter(t);
|
||||
}
|
||||
static inline UInt8 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toQuarter(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toQuarter(ExtendedDayNum(d));
|
||||
@ -408,6 +474,10 @@ struct ToMonthImpl
|
||||
{
|
||||
return time_zone.toMonth(t);
|
||||
}
|
||||
static inline UInt8 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toMonth(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toMonth(ExtendedDayNum(d));
|
||||
@ -428,6 +498,10 @@ struct ToDayOfMonthImpl
|
||||
{
|
||||
return time_zone.toDayOfMonth(t);
|
||||
}
|
||||
static inline UInt8 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toDayOfMonth(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toDayOfMonth(ExtendedDayNum(d));
|
||||
@ -448,6 +522,10 @@ struct ToDayOfWeekImpl
|
||||
{
|
||||
return time_zone.toDayOfWeek(t);
|
||||
}
|
||||
static inline UInt8 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toDayOfWeek(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toDayOfWeek(ExtendedDayNum(d));
|
||||
@ -468,6 +546,10 @@ struct ToDayOfYearImpl
|
||||
{
|
||||
return time_zone.toDayOfYear(t);
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toDayOfYear(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toDayOfYear(ExtendedDayNum(d));
|
||||
@ -488,7 +570,10 @@ struct ToHourImpl
|
||||
{
|
||||
return time_zone.toHour(t);
|
||||
}
|
||||
|
||||
static inline UInt8 execute(Int32, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
}
|
||||
static inline UInt8 execute(UInt16, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
@ -511,6 +596,11 @@ struct TimezoneOffsetImpl
|
||||
return time_zone.timezoneOffset(t);
|
||||
}
|
||||
|
||||
static inline time_t execute(Int32, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
}
|
||||
|
||||
static inline time_t execute(UInt16, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
@ -531,6 +621,10 @@ struct ToMinuteImpl
|
||||
{
|
||||
return time_zone.toMinute(t);
|
||||
}
|
||||
static inline UInt8 execute(Int32, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
}
|
||||
static inline UInt8 execute(UInt16, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
@ -551,6 +645,10 @@ struct ToSecondImpl
|
||||
{
|
||||
return time_zone.toSecond(t);
|
||||
}
|
||||
static inline UInt8 execute(Int32, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
}
|
||||
static inline UInt8 execute(UInt16, const DateLUTImpl &)
|
||||
{
|
||||
return dateIsNotSupported(name);
|
||||
@ -571,6 +669,10 @@ struct ToISOYearImpl
|
||||
{
|
||||
return time_zone.toISOYear(time_zone.toDayNum(t));
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toISOYear(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toISOYear(ExtendedDayNum(d));
|
||||
@ -591,6 +693,10 @@ struct ToStartOfISOYearImpl
|
||||
{
|
||||
return time_zone.toFirstDayNumOfISOYear(time_zone.toDayNum(t));
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(d));
|
||||
@ -611,6 +717,10 @@ struct ToISOWeekImpl
|
||||
{
|
||||
return time_zone.toISOWeek(time_zone.toDayNum(t));
|
||||
}
|
||||
static inline UInt8 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toISOWeek(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toISOWeek(ExtendedDayNum(d));
|
||||
@ -631,6 +741,10 @@ struct ToRelativeYearNumImpl
|
||||
{
|
||||
return time_zone.toYear(static_cast<time_t>(t));
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toYear(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toYear(ExtendedDayNum(d));
|
||||
@ -651,6 +765,10 @@ struct ToRelativeQuarterNumImpl
|
||||
{
|
||||
return time_zone.toRelativeQuarterNum(static_cast<time_t>(t));
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toRelativeQuarterNum(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toRelativeQuarterNum(ExtendedDayNum(d));
|
||||
@ -671,6 +789,10 @@ struct ToRelativeMonthNumImpl
|
||||
{
|
||||
return time_zone.toRelativeMonthNum(static_cast<time_t>(t));
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toRelativeMonthNum(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toRelativeMonthNum(ExtendedDayNum(d));
|
||||
@ -691,6 +813,10 @@ struct ToRelativeWeekNumImpl
|
||||
{
|
||||
return time_zone.toRelativeWeekNum(static_cast<time_t>(t));
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toRelativeWeekNum(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toRelativeWeekNum(ExtendedDayNum(d));
|
||||
@ -711,6 +837,10 @@ struct ToRelativeDayNumImpl
|
||||
{
|
||||
return time_zone.toDayNum(static_cast<time_t>(t));
|
||||
}
|
||||
static inline UInt16 execute(Int32 d, const DateLUTImpl &)
|
||||
{
|
||||
return static_cast<ExtendedDayNum>(d);
|
||||
}
|
||||
static inline UInt16 execute(UInt16 d, const DateLUTImpl &)
|
||||
{
|
||||
return static_cast<DayNum>(d);
|
||||
@ -732,6 +862,10 @@ struct ToRelativeHourNumImpl
|
||||
{
|
||||
return time_zone.toRelativeHourNum(static_cast<time_t>(t));
|
||||
}
|
||||
static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toRelativeHourNum(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toRelativeHourNum(ExtendedDayNum(d));
|
||||
@ -752,6 +886,10 @@ struct ToRelativeMinuteNumImpl
|
||||
{
|
||||
return time_zone.toRelativeMinuteNum(static_cast<time_t>(t));
|
||||
}
|
||||
static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toRelativeMinuteNum(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toRelativeMinuteNum(ExtendedDayNum(d));
|
||||
@ -772,6 +910,10 @@ struct ToRelativeSecondNumImpl
|
||||
{
|
||||
return t;
|
||||
}
|
||||
static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.fromDayNum(ExtendedDayNum(d));
|
||||
}
|
||||
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.fromDayNum(ExtendedDayNum(d));
|
||||
@ -792,6 +934,10 @@ struct ToYYYYMMImpl
|
||||
{
|
||||
return time_zone.toNumYYYYMM(t);
|
||||
}
|
||||
static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toNumYYYYMM(static_cast<ExtendedDayNum>(d));
|
||||
}
|
||||
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toNumYYYYMM(static_cast<DayNum>(d));
|
||||
@ -812,6 +958,10 @@ struct ToYYYYMMDDImpl
|
||||
{
|
||||
return time_zone.toNumYYYYMMDD(t);
|
||||
}
|
||||
static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toNumYYYYMMDD(static_cast<ExtendedDayNum>(d));
|
||||
}
|
||||
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toNumYYYYMMDD(static_cast<DayNum>(d));
|
||||
@ -832,6 +982,10 @@ struct ToYYYYMMDDhhmmssImpl
|
||||
{
|
||||
return time_zone.toNumYYYYMMDDhhmmss(t);
|
||||
}
|
||||
static inline UInt64 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toNumYYYYMMDDhhmmss(time_zone.toDate(static_cast<ExtendedDayNum>(d)));
|
||||
}
|
||||
static inline UInt64 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.toNumYYYYMMDDhhmmss(time_zone.toDate(static_cast<DayNum>(d)));
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDate32.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <Functions/CustomWeekTransforms.h>
|
||||
@ -35,7 +36,7 @@ public:
|
||||
{
|
||||
if (arguments.size() == 1)
|
||||
{
|
||||
if (!isDate(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
|
||||
if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
|
||||
throw Exception(
|
||||
"Illegal type " + arguments[0].type->getName() + " of argument of function " + getName()
|
||||
+ ". Should be a date or a date with time",
|
||||
@ -43,7 +44,7 @@ public:
|
||||
}
|
||||
else if (arguments.size() == 2)
|
||||
{
|
||||
if (!isDate(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
|
||||
if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
|
||||
throw Exception(
|
||||
"Illegal type " + arguments[0].type->getName() + " of argument of function " + getName()
|
||||
+ ". Should be a date or a date with time",
|
||||
@ -59,7 +60,7 @@ public:
|
||||
}
|
||||
else if (arguments.size() == 3)
|
||||
{
|
||||
if (!isDate(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
|
||||
if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
|
||||
throw Exception(
|
||||
"Illegal type " + arguments[0].type->getName() + " of argument of function " + getName()
|
||||
+ ". Should be a date or a date with time",
|
||||
@ -105,6 +106,9 @@ public:
|
||||
if (which.isDate())
|
||||
return CustomWeekTransformImpl<DataTypeDate, ToDataType>::execute(
|
||||
arguments, result_type, input_rows_count, Transform{});
|
||||
else if (which.isDate32())
|
||||
return CustomWeekTransformImpl<DataTypeDate32, ToDataType>::execute(
|
||||
arguments, result_type, input_rows_count, Transform{});
|
||||
else if (which.isDateTime())
|
||||
return CustomWeekTransformImpl<DataTypeDateTime, ToDataType>::execute(
|
||||
arguments, result_type, input_rows_count, Transform{});
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
#include <common/DateLUTImpl.h>
|
||||
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDate32.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
|
||||
@ -50,7 +51,11 @@ struct AddSecondsImpl
|
||||
{
|
||||
return t + delta;
|
||||
}
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED Int64 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
// use default datetime64 scale
|
||||
return (time_zone.fromDayNum(ExtendedDayNum(d)) + delta) * 1000;
|
||||
}
|
||||
static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.fromDayNum(ExtendedDayNum(d)) + delta;
|
||||
@ -71,7 +76,11 @@ struct AddMinutesImpl
|
||||
{
|
||||
return t + delta * 60;
|
||||
}
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED Int64 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
// use default datetime64 scale
|
||||
return (time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 60) * 1000;
|
||||
}
|
||||
static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 60;
|
||||
@ -91,7 +100,11 @@ struct AddHoursImpl
|
||||
{
|
||||
return t + delta * 3600;
|
||||
}
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED Int64 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
// use default datetime64 scale
|
||||
return (time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 3600) * 1000;
|
||||
}
|
||||
static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 3600;
|
||||
@ -117,6 +130,11 @@ struct AddDaysImpl
|
||||
{
|
||||
return d + delta;
|
||||
}
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED Int32 execute(Int32 d, Int64 delta, const DateLUTImpl &)
|
||||
{
|
||||
return d + delta;
|
||||
}
|
||||
};
|
||||
|
||||
struct AddWeeksImpl
|
||||
@ -124,17 +142,22 @@ struct AddWeeksImpl
|
||||
static constexpr auto name = "addWeeks";
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents<DateTime64>
|
||||
execute(DecimalUtils::DecimalComponents<DateTime64> t, Int64 delta, const DateLUTImpl & time_zone)
|
||||
execute(DecimalUtils::DecimalComponents<DateTime64> t, Int32 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return {time_zone.addWeeks(t.whole, delta), t.fractional};
|
||||
}
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone)
|
||||
static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int32 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.addWeeks(t, delta);
|
||||
}
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl &)
|
||||
static inline NO_SANITIZE_UNDEFINED UInt16 execute(UInt16 d, Int32 delta, const DateLUTImpl &)
|
||||
{
|
||||
return d + delta * 7;
|
||||
}
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED Int32 execute(Int32 d, Int32 delta, const DateLUTImpl &)
|
||||
{
|
||||
return d + delta * 7;
|
||||
}
|
||||
@ -159,6 +182,11 @@ struct AddMonthsImpl
|
||||
{
|
||||
return time_zone.addMonths(ExtendedDayNum(d), delta);
|
||||
}
|
||||
|
||||
static inline Int32 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.addMonths(ExtendedDayNum(d), delta);
|
||||
}
|
||||
};
|
||||
|
||||
struct AddQuartersImpl
|
||||
@ -166,17 +194,22 @@ struct AddQuartersImpl
|
||||
static constexpr auto name = "addQuarters";
|
||||
|
||||
static inline DecimalUtils::DecimalComponents<DateTime64>
|
||||
execute(DecimalUtils::DecimalComponents<DateTime64> t, Int64 delta, const DateLUTImpl & time_zone)
|
||||
execute(DecimalUtils::DecimalComponents<DateTime64> t, Int32 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return {time_zone.addQuarters(t.whole, delta), t.fractional};
|
||||
}
|
||||
|
||||
static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone)
|
||||
static inline UInt32 execute(UInt32 t, Int32 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.addQuarters(t, delta);
|
||||
}
|
||||
|
||||
static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone)
|
||||
static inline UInt16 execute(UInt16 d, Int32 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.addQuarters(ExtendedDayNum(d), delta);
|
||||
}
|
||||
|
||||
static inline Int32 execute(Int32 d, Int32 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.addQuarters(ExtendedDayNum(d), delta);
|
||||
}
|
||||
@ -201,6 +234,11 @@ struct AddYearsImpl
|
||||
{
|
||||
return time_zone.addYears(ExtendedDayNum(d), delta);
|
||||
}
|
||||
|
||||
static inline Int32 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return time_zone.addYears(ExtendedDayNum(d), delta);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Transform>
|
||||
@ -342,7 +380,7 @@ template <typename FieldType> struct ResultDataTypeMap {};
|
||||
template <> struct ResultDataTypeMap<UInt16> { using ResultDataType = DataTypeDate; };
|
||||
template <> struct ResultDataTypeMap<Int16> { using ResultDataType = DataTypeDate; };
|
||||
template <> struct ResultDataTypeMap<UInt32> { using ResultDataType = DataTypeDateTime; };
|
||||
template <> struct ResultDataTypeMap<Int32> { using ResultDataType = DataTypeDateTime; };
|
||||
template <> struct ResultDataTypeMap<Int32> { using ResultDataType = DataTypeDate32; };
|
||||
template <> struct ResultDataTypeMap<DateTime64> { using ResultDataType = DataTypeDateTime64; };
|
||||
template <> struct ResultDataTypeMap<Int64> { using ResultDataType = DataTypeDateTime64; };
|
||||
}
|
||||
@ -375,7 +413,7 @@ public:
|
||||
|
||||
if (arguments.size() == 2)
|
||||
{
|
||||
if (!isDate(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
|
||||
if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
|
||||
throw Exception{"Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() +
|
||||
". Should be a date or a date with time", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
|
||||
}
|
||||
@ -398,6 +436,8 @@ public:
|
||||
{
|
||||
case TypeIndex::Date:
|
||||
return resolveReturnType<DataTypeDate>(arguments);
|
||||
case TypeIndex::Date32:
|
||||
return resolveReturnType<DataTypeDate32>(arguments);
|
||||
case TypeIndex::DateTime:
|
||||
return resolveReturnType<DataTypeDateTime>(arguments);
|
||||
case TypeIndex::DateTime64:
|
||||
@ -437,16 +477,23 @@ public:
|
||||
|
||||
if constexpr (std::is_same_v<ResultDataType, DataTypeDate>)
|
||||
return std::make_shared<DataTypeDate>();
|
||||
else if constexpr (std::is_same_v<ResultDataType, DataTypeDate32>)
|
||||
return std::make_shared<DataTypeDate32>();
|
||||
else if constexpr (std::is_same_v<ResultDataType, DataTypeDateTime>)
|
||||
{
|
||||
return std::make_shared<DataTypeDateTime>(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0));
|
||||
}
|
||||
else if constexpr (std::is_same_v<ResultDataType, DataTypeDateTime64>)
|
||||
{
|
||||
// TODO (vnemkov): what if there is an overload of Transform::execute() that returns DateTime64 from DateTime or Date ?
|
||||
// Shall we use the default scale or one from optional argument ?
|
||||
const auto & datetime64_type = assert_cast<const DataTypeDateTime64 &>(*arguments[0].type);
|
||||
return std::make_shared<DataTypeDateTime64>(datetime64_type.getScale(), extractTimeZoneNameFromFunctionArguments(arguments, 2, 0));
|
||||
if (typeid_cast<const DataTypeDateTime64 *>(arguments[0].type.get()))
|
||||
{
|
||||
const auto & datetime64_type = assert_cast<const DataTypeDateTime64 &>(*arguments[0].type);
|
||||
return std::make_shared<DataTypeDateTime64>(datetime64_type.getScale(), extractTimeZoneNameFromFunctionArguments(arguments, 2, 0));
|
||||
}
|
||||
else
|
||||
{
|
||||
return std::make_shared<DataTypeDateTime64>(DataTypeDateTime64::default_scale, extractTimeZoneNameFromFunctionArguments(arguments, 2, 0));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -470,6 +517,11 @@ public:
|
||||
return DateTimeAddIntervalImpl<DataTypeDate, TransformResultDataType<DataTypeDate>, Transform>::execute(
|
||||
Transform{}, arguments, result_type);
|
||||
}
|
||||
else if (which.isDate32())
|
||||
{
|
||||
return DateTimeAddIntervalImpl<DataTypeDate32, TransformResultDataType<DataTypeDate32>, Transform>::execute(
|
||||
Transform{}, arguments, result_type);
|
||||
}
|
||||
else if (which.isDateTime())
|
||||
{
|
||||
return DateTimeAddIntervalImpl<DataTypeDateTime, TransformResultDataType<DataTypeDateTime>, Transform>::execute(
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDate32.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
@ -38,7 +39,7 @@ public:
|
||||
{
|
||||
if (arguments.size() == 1)
|
||||
{
|
||||
if (!isDate(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
|
||||
if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
|
||||
throw Exception(
|
||||
"Illegal type " + arguments[0].type->getName() + " of argument of function " + getName()
|
||||
+ ". Should be a date or a date with time",
|
||||
@ -46,7 +47,7 @@ public:
|
||||
}
|
||||
else if (arguments.size() == 2)
|
||||
{
|
||||
if (!isDate(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
|
||||
if (!isDate(arguments[0].type) && !isDate32(arguments[0].type) && !isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type))
|
||||
throw Exception(
|
||||
"Illegal type " + arguments[0].type->getName() + " of argument of function " + getName()
|
||||
+ ". Should be a date or a date with time",
|
||||
@ -57,7 +58,7 @@ public:
|
||||
"must be of type Date or DateTime. The 2nd argument (optional) must be "
|
||||
"a constant string with timezone name",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
if (isDate(arguments[0].type) && std::is_same_v<ToDataType, DataTypeDate>)
|
||||
if ((isDate(arguments[0].type) || isDate32(arguments[0].type)) && (std::is_same_v<ToDataType, DataTypeDate> || std::is_same_v<ToDataType, DataTypeDate32>))
|
||||
throw Exception(
|
||||
"The timezone argument of function " + getName() + " is allowed only when the 1st argument has the type DateTime",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
@ -103,6 +104,8 @@ public:
|
||||
|
||||
if (which.isDate())
|
||||
return DateTimeTransformImpl<DataTypeDate, ToDataType, Transform>::execute(arguments, result_type, input_rows_count);
|
||||
else if (which.isDate32())
|
||||
return DateTimeTransformImpl<DataTypeDate32, ToDataType, Transform>::execute(arguments, result_type, input_rows_count);
|
||||
else if (which.isDateTime())
|
||||
return DateTimeTransformImpl<DataTypeDateTime, ToDataType, Transform>::execute(arguments, result_type, input_rows_count);
|
||||
else if (which.isDateTime64())
|
||||
@ -146,6 +149,12 @@ public:
|
||||
== Transform::FactorTransform::execute(UInt16(right.get<UInt64>()), date_lut)
|
||||
? is_monotonic : is_not_monotonic;
|
||||
}
|
||||
else if (checkAndGetDataType<DataTypeDate32>(&type))
|
||||
{
|
||||
return Transform::FactorTransform::execute(Int32(left.get<UInt64>()), date_lut)
|
||||
== Transform::FactorTransform::execute(Int32(right.get<UInt64>()), date_lut)
|
||||
? is_monotonic : is_not_monotonic;
|
||||
}
|
||||
else
|
||||
{
|
||||
return Transform::FactorTransform::execute(UInt32(left.get<UInt64>()), date_lut)
|
||||
|
||||
@ -28,7 +28,7 @@ public:
|
||||
static constexpr auto name = or_null ? "joinGetOrNull" : "joinGet";
|
||||
|
||||
bool useDefaultImplementationForNulls() const override { return false; }
|
||||
bool useDefaultImplementationForLowCardinalityColumns() const override { return true; }
|
||||
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override;
|
||||
|
||||
@ -19,6 +19,7 @@
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
#include <Interpreters/castColumn.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Common/IPv6ToBinary.h>
|
||||
#include <Common/formatIPv6.h>
|
||||
@ -978,7 +979,8 @@ public:
|
||||
!which.isDateTime64() &&
|
||||
!which.isUInt() &&
|
||||
!which.isFloat() &&
|
||||
!which.isDecimal())
|
||||
!which.isDecimal() &&
|
||||
!which.isAggregateFunction())
|
||||
throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
@ -990,6 +992,15 @@ public:
|
||||
const IColumn * column = arguments[0].column.get();
|
||||
ColumnPtr res_column;
|
||||
|
||||
WhichDataType which(column->getDataType());
|
||||
if (which.isAggregateFunction())
|
||||
{
|
||||
const ColumnPtr to_string = castColumn(arguments[0], std::make_shared<DataTypeString>());
|
||||
const auto * str_column = checkAndGetColumn<ColumnString>(to_string.get());
|
||||
tryExecuteString(str_column, res_column);
|
||||
return res_column;
|
||||
}
|
||||
|
||||
if (tryExecuteUInt<UInt8>(column, res_column) ||
|
||||
tryExecuteUInt<UInt16>(column, res_column) ||
|
||||
tryExecuteUInt<UInt32>(column, res_column) ||
|
||||
|
||||
@ -1081,7 +1081,7 @@ public:
|
||||
const DataTypeTuple * right_tuple = checkAndGetDataType<DataTypeTuple>(arguments[1].get());
|
||||
|
||||
bool both_represented_by_number = arguments[0]->isValueRepresentedByNumber() && arguments[1]->isValueRepresentedByNumber();
|
||||
bool has_date = left.isDate() || right.isDate();
|
||||
bool has_date = left.isDateOrDate32() || right.isDateOrDate32();
|
||||
|
||||
if (!((both_represented_by_number && !has_date) /// Do not allow to compare date and number.
|
||||
|| (left.isStringOrFixedString() || right.isStringOrFixedString()) /// Everything can be compared with string by conversion.
|
||||
|
||||
@ -32,7 +32,7 @@ void registerFunctionsConversion(FunctionFactory & factory)
|
||||
factory.registerFunction<FunctionToDate>();
|
||||
/// MysQL compatibility alias.
|
||||
factory.registerFunction<FunctionToDate>("DATE", FunctionFactory::CaseInsensitive);
|
||||
|
||||
factory.registerFunction<FunctionToDate32>();
|
||||
factory.registerFunction<FunctionToDateTime>();
|
||||
factory.registerFunction<FunctionToDateTime32>();
|
||||
factory.registerFunction<FunctionToDateTime64>();
|
||||
@ -62,6 +62,7 @@ void registerFunctionsConversion(FunctionFactory & factory)
|
||||
factory.registerFunction<FunctionToFloat32OrZero>();
|
||||
factory.registerFunction<FunctionToFloat64OrZero>();
|
||||
factory.registerFunction<FunctionToDateOrZero>();
|
||||
factory.registerFunction<FunctionToDate32OrZero>();
|
||||
factory.registerFunction<FunctionToDateTimeOrZero>();
|
||||
factory.registerFunction<FunctionToDateTime64OrZero>();
|
||||
|
||||
@ -87,6 +88,7 @@ void registerFunctionsConversion(FunctionFactory & factory)
|
||||
factory.registerFunction<FunctionToFloat32OrNull>();
|
||||
factory.registerFunction<FunctionToFloat64OrNull>();
|
||||
factory.registerFunction<FunctionToDateOrNull>();
|
||||
factory.registerFunction<FunctionToDate32OrNull>();
|
||||
factory.registerFunction<FunctionToDateTimeOrNull>();
|
||||
factory.registerFunction<FunctionToDateTime64OrNull>();
|
||||
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypeFixedString.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDate32.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypeEnum.h>
|
||||
@ -670,6 +671,8 @@ struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType,
|
||||
|
||||
if constexpr (std::is_same_v<FromDataType, DataTypeDate>)
|
||||
data_to.resize(size * (strlen("YYYY-MM-DD") + 1));
|
||||
else if constexpr (std::is_same_v<FromDataType, DataTypeDate32>)
|
||||
data_to.resize(size * (strlen("YYYY-MM-DD") + 1));
|
||||
else if constexpr (std::is_same_v<FromDataType, DataTypeDateTime>)
|
||||
data_to.resize(size * (strlen("YYYY-MM-DD hh:mm:ss") + 1));
|
||||
else if constexpr (std::is_same_v<FromDataType, DataTypeDateTime64>)
|
||||
@ -751,6 +754,14 @@ inline void parseImpl<DataTypeDate>(DataTypeDate::FieldType & x, ReadBuffer & rb
|
||||
x = tmp;
|
||||
}
|
||||
|
||||
template <>
|
||||
inline void parseImpl<DataTypeDate32>(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl *)
|
||||
{
|
||||
ExtendedDayNum tmp(0);
|
||||
readDateText(tmp, rb);
|
||||
x = tmp;
|
||||
}
|
||||
|
||||
// NOTE: no need of extra overload of DateTime64, since readDateTimeText64 has different signature and that case is explicitly handled in the calling code.
|
||||
template <>
|
||||
inline void parseImpl<DataTypeDateTime>(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone)
|
||||
@ -791,6 +802,16 @@ inline bool tryParseImpl<DataTypeDate>(DataTypeDate::FieldType & x, ReadBuffer &
|
||||
return true;
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool tryParseImpl<DataTypeDate32>(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl *)
|
||||
{
|
||||
ExtendedDayNum tmp(0);
|
||||
if (!tryReadDateText(tmp, rb))
|
||||
return false;
|
||||
x = tmp;
|
||||
return true;
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool tryParseImpl<DataTypeDateTime>(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone)
|
||||
{
|
||||
@ -1215,6 +1236,7 @@ struct ConvertImpl<DataTypeFixedString, DataTypeString, Name, ConvertDefaultBeha
|
||||
|
||||
/// Declared early because used below.
|
||||
struct NameToDate { static constexpr auto name = "toDate"; };
|
||||
struct NameToDate32 { static constexpr auto name = "toDate32"; };
|
||||
struct NameToDateTime { static constexpr auto name = "toDateTime"; };
|
||||
struct NameToDateTime32 { static constexpr auto name = "toDateTime32"; };
|
||||
struct NameToDateTime64 { static constexpr auto name = "toDateTime64"; };
|
||||
@ -1890,7 +1912,7 @@ struct ToDateMonotonicity
|
||||
static IFunction::Monotonicity get(const IDataType & type, const Field & left, const Field & right)
|
||||
{
|
||||
auto which = WhichDataType(type);
|
||||
if (which.isDate() || which.isDateTime() || which.isDateTime64() || which.isInt8() || which.isInt16() || which.isUInt8() || which.isUInt16())
|
||||
if (which.isDate() || which.isDate32() || which.isDateTime() || which.isDateTime64() || which.isInt8() || which.isInt16() || which.isUInt8() || which.isUInt16())
|
||||
return {true, true, true};
|
||||
else if (
|
||||
(which.isUInt() && ((left.isNull() || left.get<UInt64>() < 0xFFFF) && (right.isNull() || right.get<UInt64>() >= 0xFFFF)))
|
||||
@ -1991,6 +2013,7 @@ using FunctionToInt256 = FunctionConvert<DataTypeInt256, NameToInt256, ToNumberM
|
||||
using FunctionToFloat32 = FunctionConvert<DataTypeFloat32, NameToFloat32, ToNumberMonotonicity<Float32>>;
|
||||
using FunctionToFloat64 = FunctionConvert<DataTypeFloat64, NameToFloat64, ToNumberMonotonicity<Float64>>;
|
||||
using FunctionToDate = FunctionConvert<DataTypeDate, NameToDate, ToDateMonotonicity>;
|
||||
using FunctionToDate32 = FunctionConvert<DataTypeDate32, NameToDate32, ToDateMonotonicity>;
|
||||
using FunctionToDateTime = FunctionConvert<DataTypeDateTime, NameToDateTime, ToDateTimeMonotonicity>;
|
||||
using FunctionToDateTime32 = FunctionConvert<DataTypeDateTime, NameToDateTime32, ToDateTimeMonotonicity>;
|
||||
using FunctionToDateTime64 = FunctionConvert<DataTypeDateTime64, NameToDateTime64, UnknownMonotonicity>;
|
||||
@ -2050,6 +2073,7 @@ struct NameToInt256OrZero { static constexpr auto name = "toInt256OrZero"; };
|
||||
struct NameToFloat32OrZero { static constexpr auto name = "toFloat32OrZero"; };
|
||||
struct NameToFloat64OrZero { static constexpr auto name = "toFloat64OrZero"; };
|
||||
struct NameToDateOrZero { static constexpr auto name = "toDateOrZero"; };
|
||||
struct NameToDate32OrZero { static constexpr auto name = "toDate32OrZero"; };
|
||||
struct NameToDateTimeOrZero { static constexpr auto name = "toDateTimeOrZero"; };
|
||||
struct NameToDateTime64OrZero { static constexpr auto name = "toDateTime64OrZero"; };
|
||||
struct NameToDecimal32OrZero { static constexpr auto name = "toDecimal32OrZero"; };
|
||||
@ -2073,6 +2097,7 @@ using FunctionToInt256OrZero = FunctionConvertFromString<DataTypeInt256, NameToI
|
||||
using FunctionToFloat32OrZero = FunctionConvertFromString<DataTypeFloat32, NameToFloat32OrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
using FunctionToFloat64OrZero = FunctionConvertFromString<DataTypeFloat64, NameToFloat64OrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
using FunctionToDateOrZero = FunctionConvertFromString<DataTypeDate, NameToDateOrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
using FunctionToDate32OrZero = FunctionConvertFromString<DataTypeDate32, NameToDate32OrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
using FunctionToDateTimeOrZero = FunctionConvertFromString<DataTypeDateTime, NameToDateTimeOrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
using FunctionToDateTime64OrZero = FunctionConvertFromString<DataTypeDateTime64, NameToDateTime64OrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
using FunctionToDecimal32OrZero = FunctionConvertFromString<DataTypeDecimal<Decimal32>, NameToDecimal32OrZero, ConvertFromStringExceptionMode::Zero>;
|
||||
@ -2096,6 +2121,7 @@ struct NameToInt256OrNull { static constexpr auto name = "toInt256OrNull"; };
|
||||
struct NameToFloat32OrNull { static constexpr auto name = "toFloat32OrNull"; };
|
||||
struct NameToFloat64OrNull { static constexpr auto name = "toFloat64OrNull"; };
|
||||
struct NameToDateOrNull { static constexpr auto name = "toDateOrNull"; };
|
||||
struct NameToDate32OrNull { static constexpr auto name = "toDate32OrNull"; };
|
||||
struct NameToDateTimeOrNull { static constexpr auto name = "toDateTimeOrNull"; };
|
||||
struct NameToDateTime64OrNull { static constexpr auto name = "toDateTime64OrNull"; };
|
||||
struct NameToDecimal32OrNull { static constexpr auto name = "toDecimal32OrNull"; };
|
||||
@ -2119,6 +2145,7 @@ using FunctionToInt256OrNull = FunctionConvertFromString<DataTypeInt256, NameToI
|
||||
using FunctionToFloat32OrNull = FunctionConvertFromString<DataTypeFloat32, NameToFloat32OrNull, ConvertFromStringExceptionMode::Null>;
|
||||
using FunctionToFloat64OrNull = FunctionConvertFromString<DataTypeFloat64, NameToFloat64OrNull, ConvertFromStringExceptionMode::Null>;
|
||||
using FunctionToDateOrNull = FunctionConvertFromString<DataTypeDate, NameToDateOrNull, ConvertFromStringExceptionMode::Null>;
|
||||
using FunctionToDate32OrNull = FunctionConvertFromString<DataTypeDate32, NameToDate32OrNull, ConvertFromStringExceptionMode::Null>;
|
||||
using FunctionToDateTimeOrNull = FunctionConvertFromString<DataTypeDateTime, NameToDateTimeOrNull, ConvertFromStringExceptionMode::Null>;
|
||||
using FunctionToDateTime64OrNull = FunctionConvertFromString<DataTypeDateTime64, NameToDateTime64OrNull, ConvertFromStringExceptionMode::Null>;
|
||||
using FunctionToDecimal32OrNull = FunctionConvertFromString<DataTypeDecimal<Decimal32>, NameToDecimal32OrNull, ConvertFromStringExceptionMode::Null>;
|
||||
|
||||
@ -163,13 +163,6 @@ public:
|
||||
arguments[0]->getName(),
|
||||
getName());
|
||||
|
||||
if (!WhichDataType(arguments[1]).isUInt64() &&
|
||||
!isTuple(arguments[1]))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of second argument of function {} must be UInt64 or tuple(...)",
|
||||
arguments[1]->getName(),
|
||||
getName());
|
||||
|
||||
return std::make_shared<DataTypeUInt8>();
|
||||
}
|
||||
|
||||
@ -189,8 +182,8 @@ public:
|
||||
auto dictionary_key_type = dictionary->getKeyType();
|
||||
|
||||
const ColumnWithTypeAndName & key_column_with_type = arguments[1];
|
||||
const auto key_column = key_column_with_type.column;
|
||||
const auto key_column_type = WhichDataType(key_column_with_type.type);
|
||||
auto key_column = key_column_with_type.column;
|
||||
auto key_column_type = key_column_with_type.type;
|
||||
|
||||
ColumnPtr range_col = nullptr;
|
||||
DataTypePtr range_col_type = nullptr;
|
||||
@ -214,7 +207,7 @@ public:
|
||||
|
||||
if (dictionary_key_type == DictionaryKeyType::simple)
|
||||
{
|
||||
if (!key_column_type.isUInt64())
|
||||
if (!WhichDataType(key_column_type).isUInt64())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Second argument of function {} must be UInt64 when dictionary is simple. Actual type {}.",
|
||||
@ -225,24 +218,39 @@ public:
|
||||
}
|
||||
else if (dictionary_key_type == DictionaryKeyType::complex)
|
||||
{
|
||||
if (!key_column_type.isTuple())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Second argument of function {} must be tuple when dictionary is complex. Actual type {}.",
|
||||
getName(),
|
||||
key_column_with_type.type->getName());
|
||||
|
||||
/// Functions in external dictionaries_loader only support full-value (not constant) columns with keys.
|
||||
ColumnPtr key_column_full = key_column->convertToFullColumnIfConst();
|
||||
key_column = key_column->convertToFullColumnIfConst();
|
||||
size_t keys_size = dictionary->getStructure().getKeysSize();
|
||||
|
||||
const auto & key_columns = typeid_cast<const ColumnTuple &>(*key_column_full).getColumnsCopy();
|
||||
const auto & key_types = static_cast<const DataTypeTuple &>(*key_column_with_type.type).getElements();
|
||||
if (!isTuple(key_column_type))
|
||||
{
|
||||
if (keys_size > 1)
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Third argument of function {} must be tuple when dictionary is complex and key contains more than 1 attribute."
|
||||
"Actual type {}.",
|
||||
getName(),
|
||||
key_column_type->getName());
|
||||
}
|
||||
else
|
||||
{
|
||||
Columns tuple_columns = {std::move(key_column)};
|
||||
key_column = ColumnTuple::create(tuple_columns);
|
||||
|
||||
DataTypes tuple_types = {key_column_type};
|
||||
key_column_type = std::make_shared<DataTypeTuple>(tuple_types);
|
||||
}
|
||||
}
|
||||
|
||||
const auto & key_columns = assert_cast<const ColumnTuple &>(*key_column).getColumnsCopy();
|
||||
const auto & key_types = assert_cast<const DataTypeTuple &>(*key_column_type).getElements();
|
||||
|
||||
return dictionary->hasKeys(key_columns, key_types);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!key_column_type.isUInt64())
|
||||
if (!WhichDataType(key_column_type).isUInt64())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Second argument of function {} must be UInt64 when dictionary is range. Actual type {}.",
|
||||
@ -346,13 +354,6 @@ public:
|
||||
Strings attribute_names = getAttributeNamesFromColumn(arguments[1].column, arguments[1].type);
|
||||
|
||||
auto dictionary = helper.getDictionary(dictionary_name);
|
||||
|
||||
if (!WhichDataType(arguments[2].type).isUInt64() && !isTuple(arguments[2].type))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of third argument of function {}, must be UInt64 or tuple(...).",
|
||||
arguments[2].type->getName(),
|
||||
getName());
|
||||
|
||||
auto dictionary_key_type = dictionary->getKeyType();
|
||||
|
||||
size_t current_arguments_index = 3;
|
||||
@ -446,18 +447,35 @@ public:
|
||||
}
|
||||
else if (dictionary_key_type == DictionaryKeyType::complex)
|
||||
{
|
||||
if (!isTuple(key_col_with_type.type))
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Third argument of function {} must be tuple when dictionary is complex. Actual type {}.",
|
||||
getName(),
|
||||
key_col_with_type.type->getName());
|
||||
|
||||
/// Functions in external dictionaries_loader only support full-value (not constant) columns with keys.
|
||||
ColumnPtr key_column_full = key_col_with_type.column->convertToFullColumnIfConst();
|
||||
ColumnPtr key_column = key_col_with_type.column->convertToFullColumnIfConst();
|
||||
DataTypePtr key_column_type = key_col_with_type.type;
|
||||
|
||||
const auto & key_columns = typeid_cast<const ColumnTuple &>(*key_column_full).getColumnsCopy();
|
||||
const auto & key_types = static_cast<const DataTypeTuple &>(*key_col_with_type.type).getElements();
|
||||
size_t keys_size = dictionary->getStructure().getKeysSize();
|
||||
|
||||
if (!isTuple(key_column_type))
|
||||
{
|
||||
if (keys_size > 1)
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Third argument of function {} must be tuple when dictionary is complex and key contains more than 1 attribute."
|
||||
"Actual type {}.",
|
||||
getName(),
|
||||
key_col_with_type.type->getName());
|
||||
}
|
||||
else
|
||||
{
|
||||
Columns tuple_columns = {std::move(key_column)};
|
||||
key_column = ColumnTuple::create(tuple_columns);
|
||||
|
||||
DataTypes tuple_types = {key_column_type};
|
||||
key_column_type = std::make_shared<DataTypeTuple>(tuple_types);
|
||||
}
|
||||
}
|
||||
|
||||
const auto & key_columns = assert_cast<const ColumnTuple &>(*key_column).getColumnsCopy();
|
||||
const auto & key_types = assert_cast<const DataTypeTuple &>(*key_column_type).getElements();
|
||||
|
||||
result = executeDictionaryRequest(
|
||||
dictionary,
|
||||
|
||||
@ -683,6 +683,8 @@ public:
|
||||
return executeType<Int64>(arguments);
|
||||
else if (which.isDate())
|
||||
return executeType<UInt16>(arguments);
|
||||
else if (which.isDate32())
|
||||
return executeType<Int32>(arguments);
|
||||
else if (which.isDateTime())
|
||||
return executeType<UInt32>(arguments);
|
||||
else if (which.isDecimal32())
|
||||
@ -986,6 +988,7 @@ private:
|
||||
else if (which.isEnum8()) executeIntType<Int8, first>(icolumn, vec_to);
|
||||
else if (which.isEnum16()) executeIntType<Int16, first>(icolumn, vec_to);
|
||||
else if (which.isDate()) executeIntType<UInt16, first>(icolumn, vec_to);
|
||||
else if (which.isDate32()) executeIntType<Int32, first>(icolumn, vec_to);
|
||||
else if (which.isDateTime()) executeIntType<UInt32, first>(icolumn, vec_to);
|
||||
/// TODO: executeIntType() for Decimal32/64 leads to incompatible result
|
||||
else if (which.isDecimal32()) executeBigIntType<Decimal32, first>(icolumn, vec_to);
|
||||
|
||||
@ -607,6 +607,8 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
template <typename JSONParser>
|
||||
class JSONExtractRawImpl;
|
||||
|
||||
/// Nodes of the extract tree. We need the extract tree to extract from JSON complex values containing array, tuples or nullables.
|
||||
template <typename JSONParser>
|
||||
@ -691,7 +693,10 @@ struct JSONExtractTree
|
||||
public:
|
||||
bool insertResultToColumn(IColumn & dest, const Element & element) override
|
||||
{
|
||||
return JSONExtractStringImpl<JSONParser>::insertResultToColumn(dest, element, {});
|
||||
if (element.isString())
|
||||
return JSONExtractStringImpl<JSONParser>::insertResultToColumn(dest, element, {});
|
||||
else
|
||||
return JSONExtractRawImpl<JSONParser>::insertResultToColumn(dest, element, {});
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -755,6 +755,7 @@ struct GenericValueSource : public ValueSourceImpl<GenericValueSource>
|
||||
{
|
||||
using Slice = GenericValueSlice;
|
||||
using SinkType = GenericArraySink;
|
||||
using Column = IColumn;
|
||||
|
||||
const IColumn * column;
|
||||
size_t total_rows;
|
||||
|
||||
@ -113,7 +113,8 @@ public:
|
||||
|
||||
virtual ~IFunctionBase() = default;
|
||||
|
||||
virtual ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, bool dry_run = false) const
|
||||
virtual ColumnPtr execute(
|
||||
const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, bool dry_run = false) const
|
||||
{
|
||||
return prepare(arguments)->execute(arguments, result_type, input_rows_count, dry_run);
|
||||
}
|
||||
@ -161,7 +162,8 @@ public:
|
||||
* Arguments are passed without modifications, useDefaultImplementationForNulls, useDefaultImplementationForConstants,
|
||||
* useDefaultImplementationForLowCardinality are not applied.
|
||||
*/
|
||||
virtual ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & /* arguments */, const DataTypePtr & /* result_type */) const { return nullptr; }
|
||||
virtual ColumnPtr getConstantResultForNonConstArguments(
|
||||
const ColumnsWithTypeAndName & /* arguments */, const DataTypePtr & /* result_type */) const { return nullptr; }
|
||||
|
||||
/** Function is called "injective" if it returns different result for different values of arguments.
|
||||
* Example: hex, negate, tuple...
|
||||
@ -358,6 +360,10 @@ public:
|
||||
*/
|
||||
virtual bool useDefaultImplementationForConstants() const { return false; }
|
||||
|
||||
/** Some arguments could remain constant during this implementation.
|
||||
*/
|
||||
virtual ColumnNumbers getArgumentsThatAreAlwaysConstant() const { return {}; }
|
||||
|
||||
/** If function arguments has single low cardinality column and all other arguments are constants, call function on nested column.
|
||||
* Otherwise, convert all low cardinality columns to ordinary columns.
|
||||
* Returns ColumnLowCardinality if at least one argument is ColumnLowCardinality.
|
||||
@ -367,10 +373,6 @@ public:
|
||||
/// If it isn't, will convert all ColumnLowCardinality arguments to full columns.
|
||||
virtual bool canBeExecutedOnLowCardinalityDictionary() const { return true; }
|
||||
|
||||
/** Some arguments could remain constant during this implementation.
|
||||
*/
|
||||
virtual ColumnNumbers getArgumentsThatAreAlwaysConstant() const { return {}; }
|
||||
|
||||
/** True if function can be called on default arguments (include Nullable's) and won't throw.
|
||||
* Counterexample: modulo(0, 0)
|
||||
*/
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include <Columns/IColumn.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
#include <Common/SipHash.h>
|
||||
#include <common/range.h>
|
||||
|
||||
@ -40,6 +41,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
|
||||
@ -304,6 +306,13 @@ void PointInPolygonWithGrid<CoordinateType>::calcGridAttributes(
|
||||
y_scale = 1 / cell_height;
|
||||
x_shift = -min_corner.x();
|
||||
y_shift = -min_corner.y();
|
||||
|
||||
if (!(isFinite(x_scale)
|
||||
&& isFinite(y_scale)
|
||||
&& isFinite(x_shift)
|
||||
&& isFinite(y_shift)
|
||||
&& isFinite(grid_size)))
|
||||
throw Exception("Polygon is not valid: bounding box is unbounded", ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
template <typename CoordinateType>
|
||||
@ -358,7 +367,7 @@ bool PointInPolygonWithGrid<CoordinateType>::contains(CoordinateType x, Coordina
|
||||
if (has_empty_bound)
|
||||
return false;
|
||||
|
||||
if (std::isnan(x) || std::isnan(y))
|
||||
if (!isFinite(x) || !isFinite(y))
|
||||
return false;
|
||||
|
||||
CoordinateType float_row = (y + y_shift) * y_scale;
|
||||
|
||||
@ -41,6 +41,9 @@ public:
|
||||
String getName() const override { return name; }
|
||||
size_t getNumberOfArguments() const override { return 2; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
if (!isString(arguments[0].type))
|
||||
@ -65,9 +68,7 @@ public:
|
||||
const ColumnConst * column_tld_list_name = checkAndGetColumnConstStringOrFixedString(arguments[1].column.get());
|
||||
FirstSignificantSubdomainCustomLookup tld_lookup(column_tld_list_name->getValue<String>());
|
||||
|
||||
/// FIXME: convertToFullColumnIfConst() is suboptimal
|
||||
auto column = arguments[0].column->convertToFullColumnIfConst();
|
||||
if (const ColumnString * col = checkAndGetColumn<ColumnString>(*column))
|
||||
if (const ColumnString * col = checkAndGetColumn<ColumnString>(*arguments[0].column))
|
||||
{
|
||||
auto col_res = ColumnString::create();
|
||||
vector(tld_lookup, col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets());
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
#include <Columns/ColumnNullable.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnFixedString.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Columns/ColumnMap.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
@ -110,6 +111,9 @@ private:
|
||||
static bool matchKeyToIndexString(const IColumn & data, const Offsets & offsets,
|
||||
const ColumnsWithTypeAndName & arguments, PaddedPODArray<UInt64> & matched_idxs);
|
||||
|
||||
static bool matchKeyToIndexFixedString(const IColumn & data, const Offsets & offsets,
|
||||
const ColumnsWithTypeAndName & arguments, PaddedPODArray<UInt64> & matched_idxs);
|
||||
|
||||
static bool matchKeyToIndexStringConst(const IColumn & data, const Offsets & offsets,
|
||||
const Field & index, PaddedPODArray<UInt64> & matched_idxs);
|
||||
|
||||
@ -767,6 +771,19 @@ struct MatcherString
|
||||
}
|
||||
};
|
||||
|
||||
struct MatcherFixedString
|
||||
{
|
||||
const ColumnFixedString & data;
|
||||
const ColumnFixedString & index;
|
||||
|
||||
bool match(size_t row_data, size_t row_index) const
|
||||
{
|
||||
auto data_ref = data.getDataAt(row_data);
|
||||
auto index_ref = index.getDataAt(row_index);
|
||||
return memequalSmallAllowOverflow15(index_ref.data, index_ref.size, data_ref.data, data_ref.size);
|
||||
}
|
||||
};
|
||||
|
||||
struct MatcherStringConst
|
||||
{
|
||||
const ColumnString & data;
|
||||
@ -863,6 +880,23 @@ bool FunctionArrayElement::matchKeyToIndexString(
|
||||
return true;
|
||||
}
|
||||
|
||||
bool FunctionArrayElement::matchKeyToIndexFixedString(
|
||||
const IColumn & data, const Offsets & offsets,
|
||||
const ColumnsWithTypeAndName & arguments, PaddedPODArray<UInt64> & matched_idxs)
|
||||
{
|
||||
const auto * index_string = checkAndGetColumn<ColumnFixedString>(arguments[1].column.get());
|
||||
if (!index_string)
|
||||
return false;
|
||||
|
||||
const auto * data_string = checkAndGetColumn<ColumnFixedString>(&data);
|
||||
if (!data_string)
|
||||
return false;
|
||||
|
||||
MatcherFixedString matcher{*data_string, *index_string};
|
||||
executeMatchKeyToIndex(offsets, matched_idxs, matcher);
|
||||
return true;
|
||||
}
|
||||
|
||||
template <typename DataType>
|
||||
bool FunctionArrayElement::matchKeyToIndexNumberConst(
|
||||
const IColumn & data, const Offsets & offsets,
|
||||
@ -922,8 +956,10 @@ bool FunctionArrayElement::matchKeyToIndex(
|
||||
|| matchKeyToIndexNumber<Int64>(data, offsets, arguments, matched_idxs)
|
||||
|| matchKeyToIndexNumber<Int128>(data, offsets, arguments, matched_idxs)
|
||||
|| matchKeyToIndexNumber<Int256>(data, offsets, arguments, matched_idxs)
|
||||
|| matchKeyToIndexNumber<UInt256>(data, offsets, arguments, matched_idxs)
|
||||
|| matchKeyToIndexNumber<UUID>(data, offsets, arguments, matched_idxs)
|
||||
|| matchKeyToIndexString(data, offsets, arguments, matched_idxs);
|
||||
|| matchKeyToIndexString(data, offsets, arguments, matched_idxs)
|
||||
|| matchKeyToIndexFixedString(data, offsets, arguments, matched_idxs);
|
||||
}
|
||||
|
||||
bool FunctionArrayElement::matchKeyToIndexConst(
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDate32.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
@ -407,6 +408,9 @@ ColumnPtr FunctionArrayIntersect::executeImpl(const ColumnsWithTypeAndName & arg
|
||||
using DateMap = ClearableHashMapWithStackMemory<DataTypeDate::FieldType,
|
||||
size_t, DefaultHash<DataTypeDate::FieldType>, INITIAL_SIZE_DEGREE>;
|
||||
|
||||
using Date32Map = ClearableHashMapWithStackMemory<DataTypeDate32::FieldType,
|
||||
size_t, DefaultHash<DataTypeDate32::FieldType>, INITIAL_SIZE_DEGREE>;
|
||||
|
||||
using DateTimeMap = ClearableHashMapWithStackMemory<
|
||||
DataTypeDateTime::FieldType, size_t,
|
||||
DefaultHash<DataTypeDateTime::FieldType>, INITIAL_SIZE_DEGREE>;
|
||||
@ -421,6 +425,8 @@ ColumnPtr FunctionArrayIntersect::executeImpl(const ColumnsWithTypeAndName & arg
|
||||
|
||||
if (which.isDate())
|
||||
result_column = execute<DateMap, ColumnVector<DataTypeDate::FieldType>, true>(arrays, std::move(column));
|
||||
else if (which.isDate32())
|
||||
result_column = execute<Date32Map, ColumnVector<DataTypeDate32::FieldType>, true>(arrays, std::move(column));
|
||||
else if (which.isDateTime())
|
||||
result_column = execute<DateTimeMap, ColumnVector<DataTypeDateTime::FieldType>, true>(arrays, std::move(column));
|
||||
else if (which.isString())
|
||||
|
||||
@ -21,6 +21,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int INCORRECT_DATA;
|
||||
}
|
||||
|
||||
namespace
|
||||
@ -79,11 +80,14 @@ public:
|
||||
const double lat = col_lat->getFloat64(row);
|
||||
const UInt8 res = col_res->getUInt(row);
|
||||
|
||||
GeoCoord coord;
|
||||
coord.lon = degsToRads(lon);
|
||||
LatLng coord;
|
||||
coord.lng = degsToRads(lon);
|
||||
coord.lat = degsToRads(lat);
|
||||
|
||||
H3Index hindex = geoToH3(&coord, res);
|
||||
H3Index hindex;
|
||||
H3Error err = latLngToCell(&coord, res, &hindex);
|
||||
if (err)
|
||||
throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect coordinates latitude: {}, longitude: {}, error: {}", coord.lat, coord.lng, err);
|
||||
|
||||
dst_data[row] = hindex;
|
||||
}
|
||||
|
||||
@ -66,7 +66,7 @@ public:
|
||||
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
// Numerical constant is 180 degrees / pi / Earth radius, Earth radius is from h3 sources
|
||||
Float64 res = 8.99320592271288084e-6 * edgeLengthM(resolution);
|
||||
Float64 res = 8.99320592271288084e-6 * getHexagonEdgeLengthAvgM(resolution);
|
||||
|
||||
dst_data[row] = res;
|
||||
}
|
||||
|
||||
@ -70,7 +70,7 @@ public:
|
||||
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
|
||||
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
Float64 res = edgeLengthM(resolution);
|
||||
Float64 res = getHexagonEdgeLengthAvgM(resolution);
|
||||
|
||||
dst_data[row] = res;
|
||||
}
|
||||
|
||||
@ -59,7 +59,7 @@ public:
|
||||
{
|
||||
const UInt64 hindex = col_hindex->getUInt(row);
|
||||
|
||||
UInt8 res = h3GetBaseCell(hindex);
|
||||
UInt8 res = getBaseCellNumber(hindex);
|
||||
|
||||
dst_data[row] = res;
|
||||
}
|
||||
|
||||
@ -59,7 +59,7 @@ public:
|
||||
{
|
||||
const UInt64 hindex = col_hindex->getUInt(row);
|
||||
|
||||
UInt8 res = h3GetResolution(hindex);
|
||||
UInt8 res = getResolution(hindex);
|
||||
|
||||
dst_data[row] = res;
|
||||
}
|
||||
|
||||
@ -65,7 +65,7 @@ public:
|
||||
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
|
||||
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
Float64 res = hexAreaM2(resolution);
|
||||
Float64 res = getHexagonAreaAvgM2(resolution);
|
||||
|
||||
dst_data[row] = res;
|
||||
}
|
||||
|
||||
@ -67,7 +67,7 @@ public:
|
||||
const UInt64 hindex_origin = col_hindex_origin->getUInt(row);
|
||||
const UInt64 hindex_dest = col_hindex_dest->getUInt(row);
|
||||
|
||||
UInt8 res = h3IndexesAreNeighbors(hindex_origin, hindex_dest);
|
||||
UInt8 res = areNeighborCells(hindex_origin, hindex_dest);
|
||||
|
||||
dst_data[row] = res;
|
||||
}
|
||||
|
||||
@ -59,7 +59,7 @@ public:
|
||||
{
|
||||
const UInt64 hindex = col_hindex->getUInt(row);
|
||||
|
||||
UInt8 is_valid = h3IsValid(hindex) == 0 ? 0 : 1;
|
||||
UInt8 is_valid = isValidCell(hindex) == 0 ? 0 : 1;
|
||||
|
||||
dst_data[row] = is_valid;
|
||||
}
|
||||
|
||||
@ -84,14 +84,14 @@ public:
|
||||
throw Exception("The argument 'resolution' (" + toString(child_resolution) + ") of function " + getName()
|
||||
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
const size_t vec_size = maxH3ToChildrenSize(parent_hindex, child_resolution);
|
||||
const size_t vec_size = cellToChildrenSize(parent_hindex, child_resolution);
|
||||
if (vec_size > MAX_ARRAY_SIZE)
|
||||
throw Exception("The result of function" + getName()
|
||||
+ " (array of " + toString(vec_size) + " elements) will be too large with resolution argument = "
|
||||
+ toString(child_resolution), ErrorCodes::TOO_LARGE_ARRAY_SIZE);
|
||||
|
||||
hindex_vec.resize(vec_size);
|
||||
h3ToChildren(parent_hindex, child_resolution, hindex_vec.data());
|
||||
cellToChildren(parent_hindex, child_resolution, hindex_vec.data());
|
||||
|
||||
dst_data.reserve(dst_data.size() + vec_size);
|
||||
for (auto hindex : hindex_vec)
|
||||
|
||||
@ -74,7 +74,7 @@ public:
|
||||
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
|
||||
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
UInt64 res = h3ToParent(hindex, resolution);
|
||||
UInt64 res = cellToParent(hindex, resolution);
|
||||
|
||||
dst_data[row] = res;
|
||||
}
|
||||
|
||||
@ -66,7 +66,7 @@ public:
|
||||
{
|
||||
const UInt64 hindex = col_hindex->getUInt(i);
|
||||
|
||||
if (!h3IsValid(hindex))
|
||||
if (!isValidCell(hindex))
|
||||
{
|
||||
throw Exception("Invalid H3 index: " + std::to_string(hindex), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user