Merge branch 'master' into memory-leak

This commit is contained in:
Sema Checherinda 2023-06-23 15:35:53 +02:00 committed by GitHub
commit 977cd03cf2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
168 changed files with 2042 additions and 1180 deletions

View File

@ -146,7 +146,7 @@ add_contrib (amqpcpp-cmake AMQP-CPP) # requires: libuv
add_contrib (cassandra-cmake cassandra) # requires: libuv
if (NOT OS_DARWIN)
add_contrib (curl-cmake curl)
add_contrib (azure-cmake azure)
add_contrib (azure-cmake azure) # requires: curl
add_contrib (sentry-native-cmake sentry-native) # requires: curl
endif()
add_contrib (fmtlib-cmake fmtlib)
@ -157,7 +157,7 @@ add_contrib (librdkafka-cmake librdkafka) # requires: libgsasl
add_contrib (nats-io-cmake nats-io)
add_contrib (isa-l-cmake isa-l)
add_contrib (libhdfs3-cmake libhdfs3) # requires: google-protobuf, krb5, isa-l
add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift/avro/arrow/libhdfs3
add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift, avro, arrow, libhdfs3
add_contrib (cppkafka-cmake cppkafka)
add_contrib (libpqxx-cmake libpqxx)
add_contrib (libpq-cmake libpq)

View File

@ -116,43 +116,79 @@ configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/A
# ARROW_ORC + adapters/orc/CMakefiles
set(ORC_SRCS
"${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.h"
"${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.cc"
"${ORC_SOURCE_SRC_DIR}/sargs/Literal.cc"
"${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.cc"
"${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.cc"
"${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.cc"
"${ORC_SOURCE_SRC_DIR}/sargs/TruthValue.cc"
"${ORC_SOURCE_SRC_DIR}/Exceptions.cc"
"${ORC_SOURCE_SRC_DIR}/OrcFile.cc"
"${ORC_SOURCE_SRC_DIR}/Reader.cc"
"${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc"
"${ORC_SOURCE_SRC_DIR}/Adaptor.cc"
"${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in"
"${ORC_SOURCE_SRC_DIR}/BlockBuffer.cc"
"${ORC_SOURCE_SRC_DIR}/BlockBuffer.hh"
"${ORC_SOURCE_SRC_DIR}/BloomFilter.cc"
"${ORC_SOURCE_SRC_DIR}/BloomFilter.hh"
"${ORC_SOURCE_SRC_DIR}/Bpacking.hh"
"${ORC_SOURCE_SRC_DIR}/BpackingDefault.cc"
"${ORC_SOURCE_SRC_DIR}/BpackingDefault.hh"
"${ORC_SOURCE_SRC_DIR}/ByteRLE.cc"
"${ORC_SOURCE_SRC_DIR}/ByteRLE.hh"
"${ORC_SOURCE_SRC_DIR}/CMakeLists.txt"
"${ORC_SOURCE_SRC_DIR}/ColumnPrinter.cc"
"${ORC_SOURCE_SRC_DIR}/ColumnReader.cc"
"${ORC_SOURCE_SRC_DIR}/ColumnReader.hh"
"${ORC_SOURCE_SRC_DIR}/ColumnWriter.cc"
"${ORC_SOURCE_SRC_DIR}/ColumnWriter.hh"
"${ORC_SOURCE_SRC_DIR}/Common.cc"
"${ORC_SOURCE_SRC_DIR}/Compression.cc"
"${ORC_SOURCE_SRC_DIR}/Compression.hh"
"${ORC_SOURCE_SRC_DIR}/ConvertColumnReader.cc"
"${ORC_SOURCE_SRC_DIR}/ConvertColumnReader.hh"
"${ORC_SOURCE_SRC_DIR}/CpuInfoUtil.cc"
"${ORC_SOURCE_SRC_DIR}/CpuInfoUtil.hh"
"${ORC_SOURCE_SRC_DIR}/Dispatch.hh"
"${ORC_SOURCE_SRC_DIR}/Exceptions.cc"
"${ORC_SOURCE_SRC_DIR}/Int128.cc"
"${ORC_SOURCE_SRC_DIR}/LzoDecompressor.cc"
"${ORC_SOURCE_SRC_DIR}/LzoDecompressor.hh"
"${ORC_SOURCE_SRC_DIR}/MemoryPool.cc"
"${ORC_SOURCE_SRC_DIR}/Murmur3.cc"
"${ORC_SOURCE_SRC_DIR}/Murmur3.hh"
"${ORC_SOURCE_SRC_DIR}/Options.hh"
"${ORC_SOURCE_SRC_DIR}/OrcFile.cc"
"${ORC_SOURCE_SRC_DIR}/RLE.cc"
"${ORC_SOURCE_SRC_DIR}/RLE.hh"
"${ORC_SOURCE_SRC_DIR}/RLEV2Util.cc"
"${ORC_SOURCE_SRC_DIR}/RLEV2Util.hh"
"${ORC_SOURCE_SRC_DIR}/RLEv1.cc"
"${ORC_SOURCE_SRC_DIR}/RLEv1.hh"
"${ORC_SOURCE_SRC_DIR}/RLEv2.hh"
"${ORC_SOURCE_SRC_DIR}/Reader.cc"
"${ORC_SOURCE_SRC_DIR}/Reader.hh"
"${ORC_SOURCE_SRC_DIR}/RleDecoderV2.cc"
"${ORC_SOURCE_SRC_DIR}/RleEncoderV2.cc"
"${ORC_SOURCE_SRC_DIR}/RLEV2Util.cc"
"${ORC_SOURCE_SRC_DIR}/SchemaEvolution.cc"
"${ORC_SOURCE_SRC_DIR}/SchemaEvolution.hh"
"${ORC_SOURCE_SRC_DIR}/Statistics.cc"
"${ORC_SOURCE_SRC_DIR}/Statistics.hh"
"${ORC_SOURCE_SRC_DIR}/StripeStream.cc"
"${ORC_SOURCE_SRC_DIR}/StripeStream.hh"
"${ORC_SOURCE_SRC_DIR}/Timezone.cc"
"${ORC_SOURCE_SRC_DIR}/Timezone.hh"
"${ORC_SOURCE_SRC_DIR}/TypeImpl.cc"
"${ORC_SOURCE_SRC_DIR}/TypeImpl.hh"
"${ORC_SOURCE_SRC_DIR}/Utils.hh"
"${ORC_SOURCE_SRC_DIR}/Vector.cc"
"${ORC_SOURCE_SRC_DIR}/Writer.cc"
"${ORC_SOURCE_SRC_DIR}/Adaptor.cc"
"${ORC_SOURCE_SRC_DIR}/BloomFilter.cc"
"${ORC_SOURCE_SRC_DIR}/Murmur3.cc"
"${ORC_SOURCE_SRC_DIR}/BlockBuffer.cc"
"${ORC_SOURCE_SRC_DIR}/wrap/orc-proto-wrapper.cc"
"${ORC_SOURCE_SRC_DIR}/io/InputStream.cc"
"${ORC_SOURCE_SRC_DIR}/io/InputStream.hh"
"${ORC_SOURCE_SRC_DIR}/io/OutputStream.cc"
"${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc"
"${ORC_SOURCE_SRC_DIR}/io/OutputStream.hh"
"${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.cc"
"${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.hh"
"${ORC_SOURCE_SRC_DIR}/sargs/Literal.cc"
"${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.cc"
"${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.hh"
"${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.cc"
"${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.hh"
"${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.cc"
"${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.hh"
"${ORC_SOURCE_SRC_DIR}/sargs/TruthValue.cc"
)
add_library(_orc ${ORC_SRCS})

View File

@ -1,6 +1,6 @@
option (ENABLE_AZURE_BLOB_STORAGE "Enable Azure blob storage" ${ENABLE_LIBRARIES})
if (NOT ENABLE_AZURE_BLOB_STORAGE OR BUILD_STANDALONE_KEEPER OR OS_FREEBSD OR (NOT ARCH_AMD64))
if (NOT ENABLE_AZURE_BLOB_STORAGE OR BUILD_STANDALONE_KEEPER OR OS_FREEBSD)
message(STATUS "Not using Azure blob storage")
return()
endif()

View File

@ -1,11 +1,11 @@
if(NOT ARCH_AARCH64 AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE AND NOT ARCH_S390X)
if(NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE AND NOT ARCH_S390X)
option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES})
elseif(ENABLE_HDFS)
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration")
endif()
if(NOT ENABLE_HDFS)
message(STATUS "Not using hdfs")
message(STATUS "Not using HDFS")
return()
endif()

2
contrib/orc vendored

@ -1 +1 @@
Subproject commit c5d7755ba0b9a95631c8daea4d094101f26ec761
Subproject commit 568d1d60c250af1890f226c182bc15bd8cc94cf1

View File

@ -0,0 +1,19 @@
---
sidebar_position: 1
sidebar_label: 2023
---
# 2023 Changelog
### ClickHouse release v23.3.5.9-lts (f5fbc2fd2b3) FIXME as compared to v23.3.4.17-lts (2c99b73ff40)
#### Bug Fix (user-visible misbehavior in an official stable release)
* Fix broken index analysis when binary operator contains a null constant argument [#50177](https://github.com/ClickHouse/ClickHouse/pull/50177) ([Amos Bird](https://github.com/amosbird)).
* Cleanup moving parts [#50489](https://github.com/ClickHouse/ClickHouse/pull/50489) ([vdimir](https://github.com/vdimir)).
* Do not apply projection if read-in-order was enabled. [#50923](https://github.com/ClickHouse/ClickHouse/pull/50923) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Increase max array size in group bitmap [#50620](https://github.com/ClickHouse/ClickHouse/pull/50620) ([Kruglov Pavel](https://github.com/Avogar)).

View File

@ -1298,8 +1298,8 @@ For output it uses the following correspondence between ClickHouse types and BSO
| [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `\x04` array |
| [Named Tuple](/docs/en/sql-reference/data-types/tuple.md) | `\x03` document |
| [Map](/docs/en/sql-reference/data-types/map.md) | `\x03` document |
| [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `\x10` int32 |
| [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `\x05` binary, `\x00` binary subtype |
| [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `\x10` int32 |
| [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `\x05` binary, `\x00` binary subtype |
For input it uses the following correspondence between BSON types and ClickHouse types:
@ -1309,7 +1309,7 @@ For input it uses the following correspondence between BSON types and ClickHouse
| `\x02` string | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) |
| `\x03` document | [Map](/docs/en/sql-reference/data-types/map.md)/[Named Tuple](/docs/en/sql-reference/data-types/tuple.md) |
| `\x04` array | [Array](/docs/en/sql-reference/data-types/array.md)/[Tuple](/docs/en/sql-reference/data-types/tuple.md) |
| `\x05` binary, `\x00` binary subtype | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md)/[IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) |
| `\x05` binary, `\x00` binary subtype | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md)/[IPv6](/docs/en/sql-reference/data-types/ipv6.md) |
| `\x05` binary, `\x02` old binary subtype | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) |
| `\x05` binary, `\x03` old uuid subtype | [UUID](/docs/en/sql-reference/data-types/uuid.md) |
| `\x05` binary, `\x04` uuid subtype | [UUID](/docs/en/sql-reference/data-types/uuid.md) |
@ -1319,7 +1319,7 @@ For input it uses the following correspondence between BSON types and ClickHouse
| `\x0A` null value | [NULL](/docs/en/sql-reference/data-types/nullable.md) |
| `\x0D` JavaScript code | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) |
| `\x0E` symbol | [String](/docs/en/sql-reference/data-types/string.md)/[FixedString](/docs/en/sql-reference/data-types/fixedstring.md) |
| `\x10` int32 | [Int32/UInt32](/docs/en/sql-reference/data-types/int-uint.md)/[Decimal32](/docs/en/sql-reference/data-types/decimal.md)/[IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md)/[Enum8/Enum16](/docs/en/sql-reference/data-types/enum.md) |
| `\x10` int32 | [Int32/UInt32](/docs/en/sql-reference/data-types/int-uint.md)/[Decimal32](/docs/en/sql-reference/data-types/decimal.md)/[IPv4](/docs/en/sql-reference/data-types/ipv4.md)/[Enum8/Enum16](/docs/en/sql-reference/data-types/enum.md) |
| `\x12` int64 | [Int64/UInt64](/docs/en/sql-reference/data-types/int-uint.md)/[Decimal64](/docs/en/sql-reference/data-types/decimal.md)/[DateTime64](/docs/en/sql-reference/data-types/datetime64.md) |
Other BSON types are not supported. Also, it performs conversion between different integer types (for example, you can insert BSON int32 value into ClickHouse UInt8).
@ -1669,8 +1669,8 @@ The table below shows supported data types and how they match ClickHouse [data t
| `ENUM` | [Enum(8/16)](/docs/en/sql-reference/data-types/enum.md) | `ENUM` |
| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` |
| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` |
| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `UINT32` |
| `DATA` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `DATA` |
| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `UINT32` |
| `DATA` | [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `DATA` |
| `DATA` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `DATA` |
| `DATA` | [Decimal128/Decimal256](/docs/en/sql-reference/data-types/decimal.md) | `DATA` |
| `STRUCT(entries LIST(STRUCT(key Key, value Value)))` | [Map](/docs/en/sql-reference/data-types/map.md) | `STRUCT(entries LIST(STRUCT(key Key, value Value)))` |
@ -1872,8 +1872,8 @@ The table below shows supported data types and how they match ClickHouse [data t
| `long (timestamp-millis)` \** | [DateTime64(3)](/docs/en/sql-reference/data-types/datetime.md) | `long (timestamp-millis)` \** |
| `long (timestamp-micros)` \** | [DateTime64(6)](/docs/en/sql-reference/data-types/datetime.md) | `long (timestamp-micros)` \** |
| `bytes (decimal)` \** | [DateTime64(N)](/docs/en/sql-reference/data-types/datetime.md) | `bytes (decimal)` \** |
| `int` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `int` |
| `fixed(16)` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `fixed(16)` |
| `int` | [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `int` |
| `fixed(16)` | [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `fixed(16)` |
| `bytes (decimal)` \** | [Decimal(P, S)](/docs/en/sql-reference/data-types/decimal.md) | `bytes (decimal)` \** |
| `string (uuid)` \** | [UUID](/docs/en/sql-reference/data-types/uuid.md) | `string (uuid)` \** |
| `fixed(16)` | [Int128/UInt128](/docs/en/sql-reference/data-types/int-uint.md) | `fixed(16)` |
@ -2026,9 +2026,9 @@ The table below shows supported data types and how they match ClickHouse [data t
| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` |
| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` |
| `MAP` | [Map](/docs/en/sql-reference/data-types/map.md) | `MAP` |
| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `UINT32` |
| `FIXED_LENGTH_BYTE_ARRAY`, `BINARY` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `FIXED_LENGTH_BYTE_ARRAY` |
| `FIXED_LENGTH_BYTE_ARRAY`, `BINARY` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `FIXED_LENGTH_BYTE_ARRAY` |
| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `UINT32` |
| `FIXED_LENGTH_BYTE_ARRAY`, `BINARY` | [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `FIXED_LENGTH_BYTE_ARRAY` |
| `FIXED_LENGTH_BYTE_ARRAY`, `BINARY` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `FIXED_LENGTH_BYTE_ARRAY` |
Arrays can be nested and can have a value of the `Nullable` type as an argument. `Tuple` and `Map` types also can be nested.
@ -2082,7 +2082,7 @@ Special format for reading Parquet file metadata (https://parquet.apache.org/doc
- logical_type - column logical type
- compression - compression used for this column
- total_uncompressed_size - total uncompressed bytes size of the column, calculated as the sum of total_uncompressed_size of the column from all row groups
- total_compressed_size - total compressed bytes size of the column, calculated as the sum of total_compressed_size of the column from all row groups
- total_compressed_size - total compressed bytes size of the column, calculated as the sum of total_compressed_size of the column from all row groups
- space_saved - percent of space saved by compression, calculated as (1 - total_compressed_size/total_uncompressed_size).
- encodings - the list of encodings used for this column
- row_groups - the list of row groups metadata with the next structure:
@ -2229,9 +2229,9 @@ The table below shows supported data types and how they match ClickHouse [data t
| `LIST` | [Array](/docs/en/sql-reference/data-types/array.md) | `LIST` |
| `STRUCT` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `STRUCT` |
| `MAP` | [Map](/docs/en/sql-reference/data-types/map.md) | `MAP` |
| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `UINT32` |
| `FIXED_SIZE_BINARY`, `BINARY` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `FIXED_SIZE_BINARY` |
| `FIXED_SIZE_BINARY`, `BINARY` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `FIXED_SIZE_BINARY` |
| `UINT32` | [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `UINT32` |
| `FIXED_SIZE_BINARY`, `BINARY` | [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `FIXED_SIZE_BINARY` |
| `FIXED_SIZE_BINARY`, `BINARY` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `FIXED_SIZE_BINARY` |
Arrays can be nested and can have a value of the `Nullable` type as an argument. `Tuple` and `Map` types also can be nested.
@ -2297,7 +2297,7 @@ The table below shows supported data types and how they match ClickHouse [data t
| `Struct` | [Tuple](/docs/en/sql-reference/data-types/tuple.md) | `Struct` |
| `Map` | [Map](/docs/en/sql-reference/data-types/map.md) | `Map` |
| `Int` | [IPv4](/docs/en/sql-reference/data-types/int-uint.md) | `Int` |
| `Binary` | [IPv6](/docs/en/sql-reference/data-types/domains/ipv6.md) | `Binary` |
| `Binary` | [IPv6](/docs/en/sql-reference/data-types/ipv6.md) | `Binary` |
| `Binary` | [Int128/UInt128/Int256/UInt256](/docs/en/sql-reference/data-types/int-uint.md) | `Binary` |
| `Binary` | [Decimal256](/docs/en/sql-reference/data-types/decimal.md) | `Binary` |
@ -2510,7 +2510,7 @@ ClickHouse supports reading and writing [MessagePack](https://msgpack.org/) data
| `uint 64` | [DateTime64](/docs/en/sql-reference/data-types/datetime.md) | `uint 64` |
| `fixarray`, `array 16`, `array 32` | [Array](/docs/en/sql-reference/data-types/array.md)/[Tuple](/docs/en/sql-reference/data-types/tuple.md) | `fixarray`, `array 16`, `array 32` |
| `fixmap`, `map 16`, `map 32` | [Map](/docs/en/sql-reference/data-types/map.md) | `fixmap`, `map 16`, `map 32` |
| `uint 32` | [IPv4](/docs/en/sql-reference/data-types/domains/ipv4.md) | `uint 32` |
| `uint 32` | [IPv4](/docs/en/sql-reference/data-types/ipv4.md) | `uint 32` |
| `bin 8` | [String](/docs/en/sql-reference/data-types/string.md) | `bin 8` |
| `int 8` | [Enum8](/docs/en/sql-reference/data-types/enum.md) | `int 8` |
| `bin 8` | [(U)Int128/(U)Int256](/docs/en/sql-reference/data-types/int-uint.md) | `bin 8` |

View File

@ -6,32 +6,43 @@ sidebar_label: Configuration Files
# Configuration Files
ClickHouse supports multi-file configuration management. The main server configuration file is `/etc/clickhouse-server/config.xml` or `/etc/clickhouse-server/config.yaml`. Other files must be in the `/etc/clickhouse-server/config.d` directory. Note, that any configuration file can be written either in XML or YAML, but mixing formats in one file is not supported. For example, you can have main configs as `config.xml` and `users.xml` and write additional files in `config.d` and `users.d` directories in `.yaml`.
The ClickHouse server can be configured with configuration files in XML or YAML syntax. In most installation types, the ClickHouse server runs with `/etc/clickhouse-server/config.xml` as default configuration file but it is also possible to specify the location of the configuration file manually at server startup using command line option `--config-file=` or `-C`. Additional configuration files may be placed into directory `config.d/` relative to the main configuration file, for example into directory `/etc/clickhouse-server/config.d/`. Files in this directory and the main configuration are merged in a preprocessing step before the configuration is applied in ClickHouse server. Configuration files are merged in alphabetical order. To simplify updates and improve modularization, it is best practice to keep the default `config.xml` file unmodified and place additional customization into `config.d/`.
All XML files should have the same root element, usually `<clickhouse>`. As for YAML, `clickhouse:` should not be present, the parser will insert it automatically.
It is possible to mix XML and YAML configuration files, for example you could have a main configuration file `config.xml` and additional configuration files `config.d/network.xml`, `config.d/timezone.yaml` and `config.d/keeper.yaml`. Mixing XML and YAML within a single configuration file is not supported. XML configuration files should use `<clickhouse>...</clickhouse>` as top-level tag. In YAML configuration files, `clickhouse:` is optional, the parser inserts it implicitly if absent.
## Override {#override}
## Overriding Configuration {#override}
Some settings specified in the main configuration file can be overridden in other configuration files:
The merge of configuration files behaves as one intuitively expects: The contents of both files are combined recursively, children with the same name are replaced by the element of the more specific configuration file. The merge can be customized using attributes `replace` and `remove`.
- Attribute `replace` means that the element is replaced by the specified one.
- Attribute `remove` means that the element is deleted.
- The `replace` or `remove` attributes can be specified for the elements of these configuration files.
- If neither is specified, it combines the contents of elements recursively, replacing values of duplicate children.
- If `replace` is specified, it replaces the entire element with the specified one.
- If `remove` is specified, it deletes the element.
To specify that a value of an element should be replaced by the value of an environment variable, you can use attribute `from_env`.
You can also declare attributes as coming from environment variables by using `from_env="VARIABLE_NAME"`:
Example with `$MAX_QUERY_SIZE = 150000`:
```xml
<clickhouse>
<macros>
<replica from_env="REPLICA" />
<layer from_env="LAYER" />
<shard from_env="SHARD" />
</macros>
<profiles>
<default>
<max_query_size from_env="MAX_QUERY_SIZE"/>
</default>
</profiles>
</clickhouse>
```
## Substitution {#substitution}
which is equal to
``` xml
<clickhouse>
<profiles>
<default>
<max_query_size/>150000</max_query_size>
</default>
</profiles>
</clickhouse>
```
## Substituting Configuration {#substitution}
The config can also define “substitutions”. If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-include_from) element in the server config. The substitution values are specified in `/clickhouse/substitution_name` elements in this file. If a substitution specified in `incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros](../operations/server-configuration-parameters/settings.md#macros)).

View File

@ -1,9 +0,0 @@
---
slug: /en/operations/optimizing-performance/
sidebar_label: Optimizing Performance
sidebar_position: 52
---
# Optimizing Performance
- [Sampling query profiler](../../operations/optimizing-performance/sampling-query-profiler.md)

View File

@ -1975,6 +1975,10 @@ The time zone is necessary for conversions between String and DateTime formats w
<timezone>Asia/Istanbul</timezone>
```
**See also**
- [session_timezone](../settings/settings.md#session_timezone)
## tcp_port {#server_configuration_parameters-tcp_port}
Port for communicating with clients over the TCP protocol.

View File

@ -4251,6 +4251,69 @@ Default value: `0`.
Use this setting only for backward compatibility if your use cases depend on old syntax.
:::
## session_timezone {#session_timezone}
Sets the implicit time zone of the current session or query.
The implicit time zone is the time zone applied to values of type DateTime/DateTime64 which have no explicitly specified time zone.
The setting takes precedence over the globally configured (server-level) implicit time zone.
A value of '' (empty string) means that the implicit time zone of the current session or query is equal to the [server time zone](../server-configuration-parameters/settings.md#server_configuration_parameters-timezone).
You can use functions `timeZone()` and `serverTimeZone()` to get the session time zone and server time zone.
Possible values:
- Any time zone name from `system.time_zones`, e.g. `Europe/Berlin`, `UTC` or `Zulu`
Default value: `''`.
Examples:
```sql
SELECT timeZone(), serverTimeZone() FORMAT TSV
Europe/Berlin Europe/Berlin
```
```sql
SELECT timeZone(), serverTimeZone() SETTINGS session_timezone = 'Asia/Novosibirsk' FORMAT TSV
Asia/Novosibirsk Europe/Berlin
```
Assign session time zone 'America/Denver' to the inner DateTime without explicitly specified time zone:
```sql
SELECT toDateTime64(toDateTime64('1999-12-12 23:23:23.123', 3), 3, 'Europe/Zurich') SETTINGS session_timezone = 'America/Denver' FORMAT TSV
1999-12-13 07:23:23.123
```
:::warning
Not all functions that parse DateTime/DateTime64 respect `session_timezone`. This can lead to subtle errors.
See the following example and explanation.
:::
```sql
CREATE TABLE test_tz (`d` DateTime('UTC')) ENGINE = Memory AS SELECT toDateTime('2000-01-01 00:00:00', 'UTC');
SELECT *, timeZone() FROM test_tz WHERE d = toDateTime('2000-01-01 00:00:00') SETTINGS session_timezone = 'Asia/Novosibirsk'
0 rows in set.
SELECT *, timeZone() FROM test_tz WHERE d = '2000-01-01 00:00:00' SETTINGS session_timezone = 'Asia/Novosibirsk'
┌───────────────────d─┬─timeZone()───────┐
│ 2000-01-01 00:00:00 │ Asia/Novosibirsk │
└─────────────────────┴──────────────────┘
```
This happens due to different parsing pipelines:
- `toDateTime()` without explicitly given time zone used in the first `SELECT` query honors setting `session_timezone` and the global time zone.
- In the second query, a DateTime is parsed from a String, and inherits the type and time zone of the existing column`d`. Thus, setting `session_timezone` and the global time zone are not honored.
**See also**
- [timezone](../server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
## final {#final}
Automatically applies [FINAL](../../sql-reference/statements/select/from.md#final-modifier) modifier to all tables in a query, to tables where [FINAL](../../sql-reference/statements/select/from.md#final-modifier) is applicable, including joined tables and tables in sub-queries, and

View File

@ -71,11 +71,11 @@ Columns:
- 0 — Query was initiated by another query as part of distributed query execution.
- `user` ([String](../../sql-reference/data-types/string.md)) — Name of the user who initiated the current query.
- `query_id` ([String](../../sql-reference/data-types/string.md)) — ID of the query.
- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the query.
- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP address that was used to make the query.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The client port that was used to make the query.
- `initial_user` ([String](../../sql-reference/data-types/string.md)) — Name of the user who ran the initial query (for distributed query execution).
- `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution).
- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from.
- `initial_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP address that the parent query was launched from.
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The client port that was used to make the parent query.
- `initial_query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Initial query starting time (for distributed query execution).
- `initial_query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Initial query starting time with microseconds precision (for distributed query execution).

View File

@ -40,11 +40,11 @@ Columns:
- 0 — Query was initiated by another query for distributed query execution.
- `user` ([String](../../sql-reference/data-types/string.md)) — Name of the user who initiated the current query.
- `query_id` ([String](../../sql-reference/data-types/string.md)) — ID of the query.
- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the query.
- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP address that was used to make the query.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the query.
- `initial_user` ([String](../../sql-reference/data-types/string.md)) — Name of the user who ran the initial query (for distributed query execution).
- `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution).
- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from.
- `initial_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP address that the parent query was launched from.
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the parent query.
- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Interface that the query was initiated from. Possible values:
- 1 — TCP.

View File

@ -28,7 +28,7 @@ Columns:
- `profiles` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — The list of profiles set for all roles and/or users.
- `roles` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — The list of roles to which the profile is applied.
- `settings` ([Array](../../sql-reference/data-types/array.md)([Tuple](../../sql-reference/data-types/tuple.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md), [String](../../sql-reference/data-types/string.md)))) — Settings that were changed when the client logged in/out.
- `client_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — The IP address that was used to log in/out.
- `client_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — The IP address that was used to log in/out.
- `client_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The client port that was used to log in/out.
- `interface` ([Enum8](../../sql-reference/data-types/enum.md)) — The interface from which the login was initiated. Possible values:
- `TCP`

View File

@ -15,7 +15,7 @@ Columns with request parameters:
- `Finalize` — The connection is lost, no response was received.
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — The date when the event happened.
- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — The date and time when the event happened.
- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address of ZooKeeper server that was used to make the request.
- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP address of ZooKeeper server that was used to make the request.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The port of ZooKeeper server that was used to make the request.
- `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — The session ID that the ZooKeeper server sets for each connection.
- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — The ID of the request within the session. This is usually a sequential request number. It is the same for the request row and the paired `response`/`finalize` row.

View File

@ -28,6 +28,6 @@ ClickHouse data types include:
- **Nested data structures**: A [`Nested` data structure](./nested-data-structures/index.md) is like a table inside a cell
- **Tuples**: A [`Tuple` of elements](./tuple.md), each having an individual type.
- **Nullable**: [`Nullable`](./nullable.md) allows you to store a value as `NULL` when a value is "missing" (instead of the column settings its default value for the data type)
- **IP addresses**: use [`IPv4`](./domains/ipv4.md) and [`IPv6`](./domains/ipv6.md) to efficiently store IP addresses
- **IP addresses**: use [`IPv4`](./ipv4.md) and [`IPv6`](./ipv6.md) to efficiently store IP addresses
- **Geo types**: for [geographical data](./geo.md), including `Point`, `Ring`, `Polygon` and `MultiPolygon`
- **Special data types**: including [`Expression`](./special-data-types/expression.md), [`Set`](./special-data-types/set.md), [`Nothing`](./special-data-types/nothing.md) and [`Interval`](./special-data-types/interval.md)

View File

@ -1,12 +1,12 @@
---
slug: /en/sql-reference/data-types/domains/ipv4
slug: /en/sql-reference/data-types/ipv4
sidebar_position: 59
sidebar_label: IPv4
---
## IPv4
`IPv4` is a domain based on `UInt32` type and serves as a typed replacement for storing IPv4 values. It provides compact storage with the human-friendly input-output format and column type information on inspection.
IPv4 addresses. Stored in 4 bytes as UInt32.
### Basic Usage
@ -57,25 +57,6 @@ SELECT toTypeName(from), hex(from) FROM hits LIMIT 1;
└──────────────────┴───────────┘
```
Domain values are not implicitly convertible to types other than `UInt32`.
If you want to convert `IPv4` value to a string, you have to do that explicitly with `IPv4NumToString()` function:
**See Also**
``` sql
SELECT toTypeName(s), IPv4NumToString(from) as s FROM hits LIMIT 1;
```
┌─toTypeName(IPv4NumToString(from))─┬─s──────────────┐
│ String │ 183.247.232.58 │
└───────────────────────────────────┴────────────────┘
Or cast to a `UInt32` value:
``` sql
SELECT toTypeName(i), CAST(from as UInt32) as i FROM hits LIMIT 1;
```
``` text
┌─toTypeName(CAST(from, 'UInt32'))─┬──────────i─┐
│ UInt32 │ 3086477370 │
└──────────────────────────────────┴────────────┘
```
- [Functions for Working with IPv4 and IPv6 Addresses](../functions/ip-address-functions.md)

View File

@ -1,12 +1,12 @@
---
slug: /en/sql-reference/data-types/domains/ipv6
slug: /en/sql-reference/data-types/ipv6
sidebar_position: 60
sidebar_label: IPv6
---
## IPv6
`IPv6` is a domain based on `FixedString(16)` type and serves as a typed replacement for storing IPv6 values. It provides compact storage with the human-friendly input-output format and column type information on inspection.
IPv6 addresses. Stored in 16 bytes as UInt128 big-endian.
### Basic Usage
@ -57,27 +57,6 @@ SELECT toTypeName(from), hex(from) FROM hits LIMIT 1;
└──────────────────┴──────────────────────────────────┘
```
Domain values are not implicitly convertible to types other than `FixedString(16)`.
If you want to convert `IPv6` value to a string, you have to do that explicitly with `IPv6NumToString()` function:
**See Also**
``` sql
SELECT toTypeName(s), IPv6NumToString(from) as s FROM hits LIMIT 1;
```
``` text
┌─toTypeName(IPv6NumToString(from))─┬─s─────────────────────────────┐
│ String │ 2001:44c8:129:2632:33:0:252:2 │
└───────────────────────────────────┴───────────────────────────────┘
```
Or cast to a `FixedString(16)` value:
``` sql
SELECT toTypeName(i), CAST(from as FixedString(16)) as i FROM hits LIMIT 1;
```
``` text
┌─toTypeName(CAST(from, 'FixedString(16)'))─┬─i───────┐
│ FixedString(16) │ <20><><EFBFBD>
└───────────────────────────────────────────┴─────────┘
```
- [Functions for Working with IPv4 and IPv6 Addresses](../functions/ip-address-functions.md)

View File

@ -139,8 +139,8 @@ makeDateTime32(year, month, day, hour, minute, second[, fraction[, precision[, t
## timeZone
Returns the timezone of the server.
If the function is executed in the context of a distributed table, it generates a normal column with values relevant to each shard, otherwise it produces a constant value.
Returns the timezone of the current session, i.e. the value of setting [session_timezone](../../operations/settings/settings.md#session_timezone).
If the function is executed in the context of a distributed table, then it generates a normal column with values relevant to each shard, otherwise it produces a constant value.
**Syntax**
@ -156,6 +156,33 @@ Alias: `timezone`.
Type: [String](../../sql-reference/data-types/string.md).
**See also**
- [serverTimeZone](#serverTimeZone)
## serverTimeZone
Returns the timezone of the server, i.e. the value of setting [timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone).
If the function is executed in the context of a distributed table, then it generates a normal column with values relevant to each shard. Otherwise, it produces a constant value.
**Syntax**
``` sql
serverTimeZone()
```
Alias: `serverTimezone`.
**Returned value**
- Timezone.
Type: [String](../../sql-reference/data-types/string.md).
**See also**
- [timeZone](#timeZone)
## toTimeZone
Converts a date or date with time to the specified time zone. Does not change the internal value (number of unix seconds) of the data, only the value's time zone attribute and the value's string representation changes.

View File

@ -248,7 +248,7 @@ SELECT IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32);
## toIPv4(string)
An alias to `IPv4StringToNum()` that takes a string form of IPv4 address and returns value of [IPv4](../../sql-reference/data-types/domains/ipv4.md) type, which is binary equal to value returned by `IPv4StringToNum()`.
An alias to `IPv4StringToNum()` that takes a string form of IPv4 address and returns value of [IPv4](../../sql-reference/data-types/ipv4.md) type, which is binary equal to value returned by `IPv4StringToNum()`.
``` sql
WITH
@ -296,7 +296,7 @@ Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns null
## toIPv6
Converts a string form of IPv6 address to [IPv6](../../sql-reference/data-types/domains/ipv6.md) type. If the IPv6 address has an invalid format, returns an empty value.
Converts a string form of IPv6 address to [IPv6](../../sql-reference/data-types/ipv6.md) type. If the IPv6 address has an invalid format, returns an empty value.
Similar to [IPv6StringToNum](#ipv6stringtonums) function, which converts IPv6 address to binary format.
If the input string contains a valid IPv4 address, then the IPv6 equivalent of the IPv4 address is returned.
@ -315,7 +315,7 @@ toIPv6(string)
- IP address.
Type: [IPv6](../../sql-reference/data-types/domains/ipv6.md).
Type: [IPv6](../../sql-reference/data-types/ipv6.md).
**Examples**

View File

@ -114,9 +114,8 @@ This example uses one table from a sample dataset. The database is `imdb`, and
`first_name` String,
`last_name` String,
`gender` FixedString(1))
ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
ORDER BY (id, first_name, last_name, gender)
SETTINGS index_granularity = 8192
ENGINE = MergeTree
ORDER BY (id, first_name, last_name, gender);
```
#### On the destination ClickHouse system:
@ -132,9 +131,8 @@ This example uses one table from a sample dataset. The database is `imdb`, and
`first_name` String,
`last_name` String,
`gender` FixedString(1))
ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')
ORDER BY (id, first_name, last_name, gender)
SETTINGS index_granularity = 8192
ENGINE = MergeTree
ORDER BY (id, first_name, last_name, gender);
```
#### Back on the source deployment:
@ -142,7 +140,7 @@ This example uses one table from a sample dataset. The database is `imdb`, and
Insert into the new database and table created on the remote system. You will need the host, port, username, password, destination database, and destination table.
```sql
INSERT INTO FUNCTION
remoteSecure('remote.clickhouse.cloud:9440', 'imdb.actors', 'USER', 'PASSWORD', rand())
remoteSecure('remote.clickhouse.cloud:9440', 'imdb.actors', 'USER', 'PASSWORD')
SELECT * from imdb.actors
```

View File

@ -1,453 +1,6 @@
agg_functions/combinators.md query-language/agg-functions/combinators.md
agg_functions/index.md query-language/agg-functions/index.md
agg_functions/parametric_functions.md query-language/agg-functions/parametric-functions.md
agg_functions/reference.md query-language/agg-functions/reference.md
changelog/2017.md whats-new/changelog/2017.md
changelog/2018.md whats-new/changelog/2018.md
changelog/2019.md whats-new/changelog/2019.md
changelog/index.md whats-new/changelog/index.md
commercial/cloud.md https://clickhouse.com/cloud/
data_types/array.md sql-reference/data-types/array.md
data_types/boolean.md sql-reference/data-types/boolean.md
data_types/date.md sql-reference/data-types/date.md
data_types/datetime.md sql-reference/data-types/datetime.md
data_types/datetime64.md sql-reference/data-types/datetime64.md
data_types/decimal.md sql-reference/data-types/decimal.md
data_types/domains/ipv4.md sql-reference/data-types/domains/ipv4.md
data_types/domains/ipv6.md sql-reference/data-types/domains/ipv6.md
data_types/domains/overview.md sql-reference/data-types/domains/overview.md
data_types/enum.md sql-reference/data-types/enum.md
data_types/fixedstring.md sql-reference/data-types/fixedstring.md
data_types/float.md sql-reference/data-types/float.md
data_types/index.md sql-reference/data-types/index.md
data_types/int_uint.md sql-reference/data-types/int-uint.md
data_types/nested_data_structures/aggregatefunction.md sql-reference/data-types/aggregatefunction.md
data_types/nested_data_structures/index.md sql-reference/data-types/nested-data-structures/index.md
data_types/nested_data_structures/nested.md sql-reference/data-types/nested-data-structures/nested.md
data_types/nullable.md sql-reference/data-types/nullable.md
data_types/special_data_types/expression.md sql-reference/data-types/special-data-types/expression.md
data_types/special_data_types/index.md sql-reference/data-types/special-data-types/index.md
data_types/special_data_types/interval.md sql-reference/data-types/special-data-types/interval.md
data_types/special_data_types/nothing.md sql-reference/data-types/special-data-types/nothing.md
data_types/special_data_types/set.md sql-reference/data-types/special-data-types/set.md
data_types/string.md sql-reference/data-types/string.md
data_types/tuple.md sql-reference/data-types/tuple.md
data_types/uuid.md sql-reference/data-types/uuid.md
database_engines/index.md engines/database-engines/index.md
database_engines/lazy.md engines/database-engines/lazy.md
database_engines/mysql.md engines/database-engines/mysql.md
development/browse_code.md development/browse-code.md
development/build_cross_arm.md development/build-cross-arm.md
development/build_cross_osx.md development/build-cross-osx.md
development/build_osx.md development/build-osx.md
development/developer_instruction.md development/developer-instruction.md
dicts/external_dicts.md query-language/dicts/external-dicts.md
dicts/external_dicts_dict.md query-language/dicts/external-dicts-dict.md
dicts/external_dicts_dict_layout.md query-language/dicts/external-dicts-dict-layout.md
dicts/external_dicts_dict_lifetime.md query-language/dicts/external-dicts-dict-lifetime.md
dicts/external_dicts_dict_sources.md query-language/dicts/external-dicts-dict-sources.md
dicts/external_dicts_dict_structure.md query-language/dicts/external-dicts-dict-structure.md
dicts/index.md query-language/dicts/index.md
dicts/internal_dicts.md query-language/dicts/internal-dicts.md
engines/database_engines/index.md engines/database-engines/index.md
engines/database_engines/lazy.md engines/database-engines/lazy.md
engines/database_engines/mysql.md engines/database-engines/mysql.md
engines/table-engines/log-family/log-family.md engines/table-engines/log-family/index.md
engines/table_engines/index.md engines/table-engines/index.md
engines/table_engines/integrations/hdfs.md engines/table-engines/integrations/hdfs.md
engines/table_engines/integrations/index.md engines/table-engines/integrations/index.md
engines/table_engines/integrations/jdbc.md engines/table-engines/integrations/jdbc.md
engines/table_engines/integrations/kafka.md engines/table-engines/integrations/kafka.md
engines/table_engines/integrations/mysql.md engines/table-engines/integrations/mysql.md
engines/table_engines/integrations/odbc.md engines/table-engines/integrations/odbc.md
engines/table_engines/log_family/index.md engines/table-engines/log-family/index.md
engines/table_engines/log_family/log.md engines/table-engines/log-family/log.md
engines/table_engines/log_family/log_family.md engines/table-engines/log-family/log-family.md
engines/table_engines/log_family/stripelog.md engines/table-engines/log-family/stripelog.md
engines/table_engines/log_family/tinylog.md engines/table-engines/log-family/tinylog.md
engines/table_engines/mergetree_family/aggregatingmergetree.md engines/table-engines/mergetree-family/aggregatingmergetree.md
engines/table_engines/mergetree_family/collapsingmergetree.md engines/table-engines/mergetree-family/collapsingmergetree.md
engines/table_engines/mergetree_family/custom_partitioning_key.md engines/table-engines/mergetree-family/custom-partitioning-key.md
engines/table_engines/mergetree_family/graphitemergetree.md engines/table-engines/mergetree-family/graphitemergetree.md
engines/table_engines/mergetree_family/index.md engines/table-engines/mergetree-family/index.md
engines/table_engines/mergetree_family/mergetree.md engines/table-engines/mergetree-family/mergetree.md
engines/table_engines/mergetree_family/replacingmergetree.md engines/table-engines/mergetree-family/replacingmergetree.md
engines/table_engines/mergetree_family/replication.md engines/table-engines/mergetree-family/replication.md
engines/table_engines/mergetree_family/summingmergetree.md engines/table-engines/mergetree-family/summingmergetree.md
engines/table_engines/mergetree_family/versionedcollapsingmergetree.md engines/table-engines/mergetree-family/versionedcollapsingmergetree.md
engines/table_engines/special/buffer.md engines/table-engines/special/buffer.md
engines/table_engines/special/dictionary.md engines/table-engines/special/dictionary.md
engines/table_engines/special/distributed.md engines/table-engines/special/distributed.md
engines/table_engines/special/external_data.md engines/table-engines/special/external-data.md
engines/table_engines/special/file.md engines/table-engines/special/file.md
engines/table_engines/special/generate.md engines/table-engines/special/generate.md
engines/table_engines/special/index.md engines/table-engines/special/index.md
engines/table_engines/special/join.md engines/table-engines/special/join.md
engines/table_engines/special/materializedview.md engines/table-engines/special/materializedview.md
engines/table_engines/special/memory.md engines/table-engines/special/memory.md
engines/table_engines/special/merge.md engines/table-engines/special/merge.md
engines/table_engines/special/null.md engines/table-engines/special/null.md
engines/table_engines/special/set.md engines/table-engines/special/set.md
engines/table_engines/special/url.md engines/table-engines/special/url.md
engines/table_engines/special/view.md engines/table-engines/special/view.md
extended_roadmap.md whats-new/extended-roadmap.md
formats.md interfaces/formats.md
formats/capnproto.md interfaces/formats.md
formats/csv.md interfaces/formats.md
formats/csvwithnames.md interfaces/formats.md
formats/json.md interfaces/formats.md
formats/jsoncompact.md interfaces/formats.md
formats/jsoneachrow.md interfaces/formats.md
formats/native.md interfaces/formats.md
formats/null.md interfaces/formats.md
formats/pretty.md interfaces/formats.md
formats/prettycompact.md interfaces/formats.md
formats/prettycompactmonoblock.md interfaces/formats.md
formats/prettynoescapes.md interfaces/formats.md
formats/prettyspace.md interfaces/formats.md
formats/rowbinary.md interfaces/formats.md
formats/tabseparated.md interfaces/formats.md
formats/tabseparatedraw.md interfaces/formats.md
formats/tabseparatedwithnames.md interfaces/formats.md
formats/tabseparatedwithnamesandtypes.md interfaces/formats.md
formats/tskv.md interfaces/formats.md
formats/values.md interfaces/formats.md
formats/vertical.md interfaces/formats.md
formats/verticalraw.md interfaces/formats.md
formats/xml.md interfaces/formats.md
functions/arithmetic_functions.md query-language/functions/arithmetic-functions.md
functions/array_functions.md query-language/functions/array-functions.md
functions/array_join.md query-language/functions/array-join.md
functions/bit_functions.md query-language/functions/bit-functions.md
functions/bitmap_functions.md query-language/functions/bitmap-functions.md
functions/comparison_functions.md query-language/functions/comparison-functions.md
functions/conditional_functions.md query-language/functions/conditional-functions.md
functions/date_time_functions.md query-language/functions/date-time-functions.md
functions/encoding_functions.md query-language/functions/encoding-functions.md
functions/ext_dict_functions.md query-language/functions/ext-dict-functions.md
functions/hash_functions.md query-language/functions/hash-functions.md
functions/higher_order_functions.md query-language/functions/higher-order-functions.md
functions/in_functions.md query-language/functions/in-functions.md
functions/index.md query-language/functions/index.md
functions/ip_address_functions.md query-language/functions/ip-address-functions.md
functions/json_functions.md query-language/functions/json-functions.md
functions/logical_functions.md query-language/functions/logical-functions.md
functions/math_functions.md query-language/functions/math-functions.md
functions/other_functions.md query-language/functions/other-functions.md
functions/random_functions.md query-language/functions/random-functions.md
functions/rounding_functions.md query-language/functions/rounding-functions.md
functions/splitting_merging_functions.md query-language/functions/splitting-merging-functions.md
functions/string_functions.md query-language/functions/string-functions.md
functions/string_replace_functions.md query-language/functions/string-replace-functions.md
functions/string_search_functions.md query-language/functions/string-search-functions.md
functions/type_conversion_functions.md query-language/functions/type-conversion-functions.md
functions/url_functions.md query-language/functions/url-functions.md
functions/ym_dict_functions.md query-language/functions/ym-dict-functions.md
getting_started/example_datasets/amplab_benchmark.md getting-started/example-datasets/amplab-benchmark.md
getting_started/example_datasets/criteo.md getting-started/example-datasets/criteo.md
getting_started/example_datasets/index.md getting-started/example-datasets/index.md
getting_started/example_datasets/metrica.md getting-started/example-datasets/metrica.md
getting_started/example_datasets/nyc_taxi.md getting-started/example-datasets/nyc-taxi.md
getting_started/example_datasets/ontime.md getting-started/example-datasets/ontime.md
getting_started/example_datasets/star_schema.md getting-started/example-datasets/star-schema.md
getting_started/example_datasets/wikistat.md getting-started/example-datasets/wikistat.md
getting_started/index.md getting-started/index.md
getting_started/install.md getting-started/install.md
getting_started/playground.md getting-started/playground.md
getting_started/tutorial.md getting-started/tutorial.md
images/column_oriented.gif images/column-oriented.gif
images/row_oriented.gif images/row-oriented.gif
interfaces/http_interface.md interfaces/http.md
interfaces/third-party/client_libraries.md interfaces/third-party/client-libraries.md
interfaces/third-party_client_libraries.md interfaces/third-party/client-libraries.md
interfaces/third-party_gui.md interfaces/third-party/gui.md
interfaces/third_party/index.md interfaces/third-party/index.md
introduction/index.md
introduction/distinctive_features.md introduction/distinctive-features.md
introduction/features_considered_disadvantages.md introduction/distinctive-features.md
introduction/possible_silly_questions.md faq/general.md
introduction/ya_metrika_task.md introduction/history.md
operations/access_rights.md operations/access-rights.md
operations/configuration_files.md operations/configuration-files.md
operations/optimizing_performance/index.md operations/optimizing-performance/index.md
operations/optimizing_performance/sampling_query_profiler.md operations/optimizing-performance/sampling-query-profiler.md
operations/performance/sampling_query_profiler.md operations/optimizing-performance/sampling-query-profiler.md
operations/performance_test.md operations/performance-test.md
operations/server_configuration_parameters/index.md operations/server-configuration-parameters/index.md
operations/server_configuration_parameters/settings.md operations/server-configuration-parameters/settings.md
operations/server_settings/index.md operations/server-configuration-parameters/index.md
operations/server_settings/settings.md operations/server-configuration-parameters/settings.md
operations/settings/constraints_on_settings.md operations/settings/constraints-on-settings.md
operations/settings/permissions_for_queries.md operations/settings/permissions-for-queries.md
operations/settings/query_complexity.md operations/settings/query-complexity.md
operations/settings/settings_profiles.md operations/settings/settings-profiles.md
operations/settings/settings_users.md operations/settings/settings-users.md
operations/system_tables.md operations/system-tables.md
operations/table_engines/aggregatingmergetree.md engines/table-engines/mergetree-family/aggregatingmergetree.md
operations/table_engines/buffer.md engines/table-engines/special/buffer.md
operations/table_engines/collapsingmergetree.md engines/table-engines/mergetree-family/collapsingmergetree.md
operations/table_engines/custom_partitioning_key.md engines/table-engines/mergetree-family/custom-partitioning-key.md
operations/table_engines/dictionary.md engines/table-engines/special/dictionary.md
operations/table_engines/distributed.md engines/table-engines/special/distributed.md
operations/table_engines/external_data.md engines/table-engines/special/external-data.md
operations/table_engines/file.md engines/table-engines/special/file.md
operations/table_engines/generate.md engines/table-engines/special/generate.md
operations/table_engines/graphitemergetree.md engines/table-engines/mergetree-family/graphitemergetree.md
operations/table_engines/hdfs.md engines/table-engines/integrations/hdfs.md
operations/table_engines/index.md engines/table-engines/index.md
operations/table_engines/jdbc.md engines/table-engines/integrations/jdbc.md
operations/table_engines/join.md engines/table-engines/special/join.md
operations/table_engines/kafka.md engines/table-engines/integrations/kafka.md
operations/table_engines/log.md engines/table-engines/log-family/log.md
operations/table_engines/log_family.md engines/table-engines/log-family/log-family.md
operations/table_engines/materializedview.md engines/table-engines/special/materializedview.md
operations/table_engines/memory.md engines/table-engines/special/memory.md
operations/table_engines/merge.md engines/table-engines/special/merge.md
operations/table_engines/mergetree.md engines/table-engines/mergetree-family/mergetree.md
operations/table_engines/mysql.md engines/table-engines/integrations/mysql.md
operations/table_engines/null.md engines/table-engines/special/null.md
operations/table_engines/odbc.md engines/table-engines/integrations/odbc.md
operations/table_engines/replacingmergetree.md engines/table-engines/mergetree-family/replacingmergetree.md
operations/table_engines/replication.md engines/table-engines/mergetree-family/replication.md
operations/table_engines/set.md engines/table-engines/special/set.md
operations/table_engines/stripelog.md engines/table-engines/log-family/stripelog.md
operations/table_engines/summingmergetree.md engines/table-engines/mergetree-family/summingmergetree.md
operations/table_engines/tinylog.md engines/table-engines/log-family/tinylog.md
operations/table_engines/url.md engines/table-engines/special/url.md
operations/table_engines/versionedcollapsingmergetree.md engines/table-engines/mergetree-family/versionedcollapsingmergetree.md
operations/table_engines/view.md engines/table-engines/special/view.md
operations/utils/clickhouse-benchmark.md operations/utilities/clickhouse-benchmark.md
operations/utils/clickhouse-copier.md operations/utilities/clickhouse-copier.md
operations/utils/clickhouse-local.md operations/utilities/clickhouse-local.md
operations/utils/index.md operations/utilities/index.md
query_language/agg_functions/combinators.md sql-reference/aggregate-functions/combinators.md
query_language/agg_functions/index.md sql-reference/aggregate-functions/index.md
query_language/agg_functions/parametric_functions.md sql-reference/aggregate-functions/parametric-functions.md
query_language/agg_functions/reference.md sql-reference/aggregate-functions/reference.md
query_language/alter.md sql-reference/statements/alter.md
query_language/create.md sql-reference/statements/create.md
query_language/dicts/external_dicts.md sql-reference/dictionaries/external-dictionaries/external-dicts.md
query_language/dicts/external_dicts_dict.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md
query_language/dicts/external_dicts_dict_hierarchical.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md
query_language/dicts/external_dicts_dict_layout.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md
query_language/dicts/external_dicts_dict_lifetime.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md
query_language/dicts/external_dicts_dict_sources.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md
query_language/dicts/external_dicts_dict_structure.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md
query_language/dicts/index.md sql-reference/dictionaries/index.md
query_language/dicts/internal_dicts.md sql-reference/dictionaries/internal-dicts.md
query_language/functions/arithmetic_functions.md sql-reference/functions/arithmetic-functions.md
query_language/functions/array_functions.md sql-reference/functions/array-functions.md
query_language/functions/array_join.md sql-reference/functions/array-join.md
query_language/functions/bit_functions.md sql-reference/functions/bit-functions.md
query_language/functions/bitmap_functions.md sql-reference/functions/bitmap-functions.md
query_language/functions/comparison_functions.md sql-reference/functions/comparison-functions.md
query_language/functions/conditional_functions.md sql-reference/functions/conditional-functions.md
query_language/functions/date_time_functions.md sql-reference/functions/date-time-functions.md
query_language/functions/encoding_functions.md sql-reference/functions/encoding-functions.md
query_language/functions/ext_dict_functions.md sql-reference/functions/ext-dict-functions.md
query_language/functions/functions_for_nulls.md sql-reference/functions/functions-for-nulls.md
query_language/functions/geo.md sql-reference/functions/geo.md
query_language/functions/hash_functions.md sql-reference/functions/hash-functions.md
query_language/functions/higher_order_functions.md sql-reference/functions/higher-order-functions.md
query_language/functions/in_functions.md sql-reference/functions/in-functions.md
query_language/functions/index.md sql-reference/functions/index.md
query_language/functions/introspection.md sql-reference/functions/introspection.md
query_language/functions/ip_address_functions.md sql-reference/functions/ip-address-functions.md
query_language/functions/json_functions.md sql-reference/functions/json-functions.md
query_language/functions/logical_functions.md sql-reference/functions/logical-functions.md
query_language/functions/machine_learning_functions.md sql-reference/functions/machine-learning-functions.md
query_language/functions/math_functions.md sql-reference/functions/math-functions.md
query_language/functions/other_functions.md sql-reference/functions/other-functions.md
query_language/functions/random_functions.md sql-reference/functions/random-functions.md
query_language/functions/rounding_functions.md sql-reference/functions/rounding-functions.md
query_language/functions/splitting_merging_functions.md sql-reference/functions/splitting-merging-functions.md
query_language/functions/string_functions.md sql-reference/functions/string-functions.md
query_language/functions/string_replace_functions.md sql-reference/functions/string-replace-functions.md
query_language/functions/string_search_functions.md sql-reference/functions/string-search-functions.md
query_language/functions/type_conversion_functions.md sql-reference/functions/type-conversion-functions.md
query_language/functions/url_functions.md sql-reference/functions/url-functions.md
query_language/functions/uuid_functions.md sql-reference/functions/uuid-functions.md
query_language/functions/ym_dict_functions.md sql-reference/functions/ym-dict-functions.md
query_language/index.md sql-reference/index.md
query_language/insert_into.md sql-reference/statements/insert-into.md
query_language/misc.md sql-reference/statements/misc.md
query_language/operators.md sql-reference/operators.md
query_language/queries.md query-language.md
query_language/select.md sql-reference/statements/select.md
query_language/show.md sql-reference/statements/show.md
query_language/syntax.md sql-reference/syntax.md
query_language/system.md sql-reference/statements/system.md
query_language/table_functions/file.md sql-reference/table-functions/file.md
query_language/table_functions/generate.md sql-reference/table-functions/generate.md
query_language/table_functions/hdfs.md sql-reference/table-functions/hdfs.md
query_language/table_functions/index.md sql-reference/table-functions/index.md
query_language/table_functions/input.md sql-reference/table-functions/input.md
query_language/table_functions/jdbc.md sql-reference/table-functions/jdbc.md
query_language/table_functions/merge.md sql-reference/table-functions/merge.md
query_language/table_functions/mysql.md sql-reference/table-functions/mysql.md
query_language/table_functions/numbers.md sql-reference/table-functions/numbers.md
query_language/table_functions/odbc.md sql-reference/table-functions/odbc.md
query_language/table_functions/remote.md sql-reference/table-functions/remote.md
query_language/table_functions/url.md sql-reference/table-functions/url.md
roadmap.md whats-new/roadmap.md
security_changelog.md whats-new/security-changelog.md
sql-reference/data-types/domains/overview.md sql-reference/data-types/domains/index.md
sql_reference/aggregate_functions/combinators.md sql-reference/aggregate-functions/combinators.md
sql_reference/aggregate_functions/index.md sql-reference/aggregate-functions/index.md
sql_reference/aggregate_functions/parametric_functions.md sql-reference/aggregate-functions/parametric-functions.md
sql_reference/aggregate_functions/reference.md sql-reference/aggregate-functions/reference.md
sql_reference/ansi.md sql-reference/ansi.md
sql_reference/data_types/aggregatefunction.md sql-reference/data-types/aggregatefunction.md
sql_reference/data_types/array.md sql-reference/data-types/array.md
sql_reference/data_types/boolean.md sql-reference/data-types/boolean.md
sql_reference/data_types/date.md sql-reference/data-types/date.md
sql_reference/data_types/datetime.md sql-reference/data-types/datetime.md
sql_reference/data_types/datetime64.md sql-reference/data-types/datetime64.md
sql_reference/data_types/decimal.md sql-reference/data-types/decimal.md
sql_reference/data_types/domains/index.md sql-reference/data-types/domains/index.md
sql_reference/data_types/domains/ipv4.md sql-reference/data-types/domains/ipv4.md
sql_reference/data_types/domains/ipv6.md sql-reference/data-types/domains/ipv6.md
sql_reference/data_types/domains/overview.md sql-reference/data-types/domains/overview.md
sql_reference/data_types/enum.md sql-reference/data-types/enum.md
sql_reference/data_types/fixedstring.md sql-reference/data-types/fixedstring.md
sql_reference/data_types/float.md sql-reference/data-types/float.md
sql_reference/data_types/index.md sql-reference/data-types/index.md
sql_reference/data_types/int_uint.md sql-reference/data-types/int-uint.md
sql_reference/data_types/nested_data_structures/index.md sql-reference/data-types/nested-data-structures/index.md
sql_reference/data_types/nested_data_structures/nested.md sql-reference/data-types/nested-data-structures/nested.md
sql_reference/data_types/nullable.md sql-reference/data-types/nullable.md
sql_reference/data_types/simpleaggregatefunction.md sql-reference/data-types/simpleaggregatefunction.md
sql_reference/data_types/special_data_types/expression.md sql-reference/data-types/special-data-types/expression.md
sql_reference/data_types/special_data_types/index.md sql-reference/data-types/special-data-types/index.md
sql_reference/data_types/special_data_types/interval.md sql-reference/data-types/special-data-types/interval.md
sql_reference/data_types/special_data_types/nothing.md sql-reference/data-types/special-data-types/nothing.md
sql_reference/data_types/special_data_types/set.md sql-reference/data-types/special-data-types/set.md
sql_reference/data_types/string.md sql-reference/data-types/string.md
sql_reference/data_types/tuple.md sql-reference/data-types/tuple.md
sql_reference/data_types/uuid.md sql-reference/data-types/uuid.md
sql_reference/dictionaries/external_dictionaries/external_dicts.md sql-reference/dictionaries/external-dictionaries/external-dicts.md
sql_reference/dictionaries/external_dictionaries/external_dicts_dict.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md
sql_reference/dictionaries/external_dictionaries/external_dicts_dict_hierarchical.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md
sql_reference/dictionaries/external_dictionaries/external_dicts_dict_layout.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md
sql_reference/dictionaries/external_dictionaries/external_dicts_dict_lifetime.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md
sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md
sql_reference/dictionaries/external_dictionaries/external_dicts_dict_structure.md sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md
sql_reference/dictionaries/external_dictionaries/index.md sql-reference/dictionaries/external-dictionaries/index.md
sql_reference/dictionaries/index.md sql-reference/dictionaries/index.md
sql_reference/dictionaries/internal_dicts.md sql-reference/dictionaries/internal-dicts.md
sql_reference/functions/arithmetic_functions.md sql-reference/functions/arithmetic-functions.md
sql_reference/functions/array_functions.md sql-reference/functions/array-functions.md
sql_reference/functions/array_join.md sql-reference/functions/array-join.md
sql_reference/functions/bit_functions.md sql-reference/functions/bit-functions.md
sql_reference/functions/bitmap_functions.md sql-reference/functions/bitmap-functions.md
sql_reference/functions/comparison_functions.md sql-reference/functions/comparison-functions.md
sql_reference/functions/conditional_functions.md sql-reference/functions/conditional-functions.md
sql_reference/functions/date_time_functions.md sql-reference/functions/date-time-functions.md
sql_reference/functions/encoding_functions.md sql-reference/functions/encoding-functions.md
sql_reference/functions/ext_dict_functions.md sql-reference/functions/ext-dict-functions.md
sql_reference/functions/functions_for_nulls.md sql-reference/functions/functions-for-nulls.md
sql_reference/functions/geo.md sql-reference/functions/geo.md
sql_reference/functions/hash_functions.md sql-reference/functions/hash-functions.md
sql_reference/functions/higher_order_functions.md sql-reference/functions/higher-order-functions.md
sql_reference/functions/in_functions.md sql-reference/functions/in-functions.md
sql_reference/functions/index.md sql-reference/functions/index.md
sql_reference/functions/introspection.md sql-reference/functions/introspection.md
sql_reference/functions/ip_address_functions.md sql-reference/functions/ip-address-functions.md
sql_reference/functions/json_functions.md sql-reference/functions/json-functions.md
sql_reference/functions/logical_functions.md sql-reference/functions/logical-functions.md
sql_reference/functions/machine_learning_functions.md sql-reference/functions/machine-learning-functions.md
sql_reference/functions/math_functions.md sql-reference/functions/math-functions.md
sql_reference/functions/other_functions.md sql-reference/functions/other-functions.md
sql_reference/functions/random_functions.md sql-reference/functions/random-functions.md
sql_reference/functions/rounding_functions.md sql-reference/functions/rounding-functions.md
sql_reference/functions/splitting_merging_functions.md sql-reference/functions/splitting-merging-functions.md
sql_reference/functions/string_functions.md sql-reference/functions/string-functions.md
sql_reference/functions/string_replace_functions.md sql-reference/functions/string-replace-functions.md
sql_reference/functions/string_search_functions.md sql-reference/functions/string-search-functions.md
sql_reference/functions/type_conversion_functions.md sql-reference/functions/type-conversion-functions.md
sql_reference/functions/url_functions.md sql-reference/functions/url-functions.md
sql_reference/functions/uuid_functions.md sql-reference/functions/uuid-functions.md
sql_reference/functions/ym_dict_functions.md sql-reference/functions/ym-dict-functions.md
sql_reference/index.md sql-reference/index.md
sql_reference/operators.md sql-reference/operators.md
sql_reference/statements/alter.md sql-reference/statements/alter.md
sql_reference/statements/create.md sql-reference/statements/create.md
sql_reference/statements/index.md sql-reference/statements/index.md
sql_reference/statements/insert_into.md sql-reference/statements/insert-into.md
sql_reference/statements/misc.md sql-reference/statements/misc.md
sql_reference/statements/select.md sql-reference/statements/select.md
sql_reference/statements/show.md sql-reference/statements/show.md
sql_reference/statements/system.md sql-reference/statements/system.md
sql_reference/syntax.md sql-reference/syntax.md
sql_reference/table_functions/file.md sql-reference/table-functions/file.md
sql_reference/table_functions/generate.md sql-reference/table-functions/generate.md
sql_reference/table_functions/hdfs.md sql-reference/table-functions/hdfs.md
sql_reference/table_functions/index.md sql-reference/table-functions/index.md
sql_reference/table_functions/input.md sql-reference/table-functions/input.md
sql_reference/table_functions/jdbc.md sql-reference/table-functions/jdbc.md
sql_reference/table_functions/merge.md sql-reference/table-functions/merge.md
sql_reference/table_functions/mysql.md sql-reference/table-functions/mysql.md
sql_reference/table_functions/numbers.md sql-reference/table-functions/numbers.md
sql_reference/table_functions/odbc.md sql-reference/table-functions/odbc.md
sql_reference/table_functions/remote.md sql-reference/table-functions/remote.md
sql_reference/table_functions/url.md sql-reference/table-functions/url.md
system_tables.md operations/system-tables.md
system_tables/system.asynchronous_metrics.md operations/system-tables.md
system_tables/system.clusters.md operations/system-tables.md
system_tables/system.columns.md operations/system-tables.md
system_tables/system.databases.md operations/system-tables.md
system_tables/system.dictionaries.md operations/system-tables.md
system_tables/system.events.md operations/system-tables.md
system_tables/system.functions.md operations/system-tables.md
system_tables/system.merges.md operations/system-tables.md
system_tables/system.metrics.md operations/system-tables.md
system_tables/system.numbers.md operations/system-tables.md
system_tables/system.numbers_mt.md operations/system-tables.md
system_tables/system.one.md operations/system-tables.md
system_tables/system.parts.md operations/system-tables.md
system_tables/system.processes.md operations/system-tables.md
system_tables/system.replicas.md operations/system-tables.md
system_tables/system.settings.md operations/system-tables.md
system_tables/system.tables.md operations/system-tables.md
system_tables/system.zookeeper.md operations/system-tables.md
table_engines.md operations/table-engines.md
table_engines/aggregatingmergetree.md operations/table-engines/aggregatingmergetree.md
table_engines/buffer.md operations/table-engines/buffer.md
table_engines/collapsingmergetree.md operations/table-engines/collapsingmergetree.md
table_engines/custom_partitioning_key.md operations/table-engines/custom-partitioning-key.md
table_engines/dictionary.md operations/table-engines/dictionary.md
table_engines/distributed.md operations/table-engines/distributed.md
table_engines/external_data.md operations/table-engines/external-data.md
table_engines/file.md operations/table-engines/file.md
table_engines/graphitemergetree.md operations/table-engines/graphitemergetree.md
table_engines/index.md operations/table-engines/index.md
table_engines/join.md operations/table-engines/join.md
table_engines/kafka.md operations/table-engines/kafka.md
table_engines/log.md operations/table-engines/log.md
table_engines/materializedview.md operations/table-engines/materializedview.md
table_engines/memory.md operations/table-engines/memory.md
table_engines/merge.md operations/table-engines/merge.md
table_engines/mergetree.md operations/table-engines/mergetree.md
table_engines/mysql.md operations/table-engines/mysql.md
table_engines/null.md operations/table-engines/null.md
table_engines/replacingmergetree.md operations/table-engines/replacingmergetree.md
table_engines/replication.md operations/table-engines/replication.md
table_engines/set.md operations/table-engines/set.md
table_engines/summingmergetree.md operations/table-engines/summingmergetree.md
table_engines/tinylog.md operations/table-engines/tinylog.md
table_engines/view.md operations/table-engines/view.md
table_functions/file.md query-language/table-functions/file.md
table_functions/index.md query-language/table-functions/index.md
table_functions/merge.md query-language/table-functions/merge.md
table_functions/numbers.md query-language/table-functions/numbers.md
table_functions/remote.md query-language/table-functions/remote.md
utils.md operations/utils.md
utils/clickhouse-copier.md operations/utils/clickhouse-copier.md
utils/clickhouse-local.md operations/utils/clickhouse-local.md
whats_new/changelog/2017.md whats-new/changelog/2017.md
whats_new/changelog/2018.md whats-new/changelog/2018.md
whats_new/changelog/2019.md whats-new/changelog/2019.md
whats_new/changelog/index.md whats-new/changelog/index.md
whats_new/index.md whats-new/index.md
whats_new/roadmap.md whats-new/roadmap.md
whats_new/security_changelog.md whats-new/security-changelog.md
The redirects from this file were moved to the Docusaurus configuration file.
If you need to add a redirect please either open a PR in
https://github.com/clickhouse/clickhouse-docs adding the redirect to
https://github.com/ClickHouse/clickhouse-docs/blob/main/docusaurus.config.js
or open an issue in the same repo and provide the old URL and new URL to have
the redirect added.

View File

@ -1355,6 +1355,10 @@ Parameters:
<timezone>Europe/Moscow</timezone>
```
**См. также**
- [session_timezone](../settings/settings.md#session_timezone)
## tcp_port {#server_configuration_parameters-tcp_port}
Порт для взаимодействия с клиентами по протоколу TCP.

View File

@ -4127,6 +4127,63 @@ SELECT sum(number) FROM numbers(10000000000) SETTINGS partial_result_on_first_ca
Значение по умолчанию: `false`
## session_timezone {#session_timezone}
Задаёт значение часового пояса (session_timezone) по умолчанию для текущей сессии вместо [часового пояса сервера](../server-configuration-parameters/settings.md#server_configuration_parameters-timezone). То есть, все значения DateTime/DateTime64, для которых явно не задан часовой пояс, будут интерпретированы как относящиеся к указанной зоне.
При значении настройки `''` (пустая строка), будет совпадать с часовым поясом сервера.
Функции `timeZone()` and `serverTimezone()` возвращают часовой пояс текущей сессии и сервера соответственно.
Примеры:
```sql
SELECT timeZone(), serverTimezone() FORMAT TSV
Europe/Berlin Europe/Berlin
```
```sql
SELECT timeZone(), serverTimezone() SETTINGS session_timezone = 'Asia/Novosibirsk' FORMAT TSV
Asia/Novosibirsk Europe/Berlin
```
```sql
SELECT toDateTime64(toDateTime64('1999-12-12 23:23:23.123', 3), 3, 'Europe/Zurich') SETTINGS session_timezone = 'America/Denver' FORMAT TSV
1999-12-13 07:23:23.123
```
Возможные значения:
- Любая зона из `system.time_zones`, например `Europe/Berlin`, `UTC` или `Zulu`
Значение по умолчанию: `''`.
:::warning
Иногда при формировании значений типа `DateTime` и `DateTime64` параметр `session_timezone` может быть проигнорирован.
Это может привести к путанице. Пример и пояснение см. ниже.
:::
```sql
CREATE TABLE test_tz (`d` DateTime('UTC')) ENGINE = Memory AS SELECT toDateTime('2000-01-01 00:00:00', 'UTC');
SELECT *, timezone() FROM test_tz WHERE d = toDateTime('2000-01-01 00:00:00') SETTINGS session_timezone = 'Asia/Novosibirsk'
0 rows in set.
SELECT *, timezone() FROM test_tz WHERE d = '2000-01-01 00:00:00' SETTINGS session_timezone = 'Asia/Novosibirsk'
┌───────────────────d─┬─timezone()───────┐
│ 2000-01-01 00:00:00 │ Asia/Novosibirsk │
└─────────────────────┴──────────────────┘
```
Это происходит из-за различного происхождения значения, используемого для сравнения:
- В первом запросе функция `toDateTime()`, создавая значение типа `DateTime`, принимает во внимание параметр `session_timezone` из контекста запроса;
- Во втором запросе `DateTime` формируется из строки неявно, наследуя тип колонки `d` (в том числе и числовой пояс), и параметр `session_timezone` игнорируется.
**Смотрите также**
- [timezone](../server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
## rename_files_after_processing
- **Тип:** Строка

View File

@ -69,11 +69,11 @@ ClickHouse не удаляет данные из таблица автомати
- 0 — запрос был инициирован другим запросом при выполнении распределенного запроса.
- `user` ([String](../../sql-reference/data-types/string.md)) — пользователь, запустивший текущий запрос.
- `query_id` ([String](../../sql-reference/data-types/string.md)) — ID запроса.
- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел запрос.
- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP адрес, с которого пришел запрос.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт, с которого клиент сделал запрос
- `initial_user` ([String](../../sql-reference/data-types/string.md)) — пользователь, запустивший первоначальный запрос (для распределенных запросов).
- `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — ID родительского запроса.
- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел родительский запрос.
- `initial_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP адрес, с которого пришел родительский запрос.
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт, с которого клиент сделал родительский запрос.
- `initial_query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала обработки запроса (для распределенных запросов).
- `initial_query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время начала обработки запроса с точностью до микросекунд (для распределенных запросов).

View File

@ -39,11 +39,11 @@ ClickHouse не удаляет данные из таблицы автомати
- 0 — запрос был инициирован другим запросом при распределенном запросе.
- `user` ([String](../../sql-reference/data-types/string.md)) — пользователь, запустивший текущий запрос.
- `query_id` ([String](../../sql-reference/data-types/string.md)) — ID запроса.
- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел запрос.
- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP адрес, с которого пришел запрос.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — порт, с которого пришел запрос.
- `initial_user` ([String](../../sql-reference/data-types/string.md)) — пользователь, запустивший первоначальный запрос (для распределенных запросов).
- `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — ID родительского запроса.
- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел родительский запрос.
- `initial_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP адрес, с которого пришел родительский запрос.
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — порт, пришел родительский запрос.
- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — интерфейс, с которого ушёл запрос. Возможные значения:
- 1 — TCP.

View File

@ -27,7 +27,7 @@ slug: /ru/operations/system-tables/session_log
- `profiles` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — список профилей, установленных для всех ролей и (или) пользователей.
- `roles` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — список ролей, к которым применяется данный профиль.
- `settings` ([Array](../../sql-reference/data-types/array.md)([Tuple](../../sql-reference/data-types/tuple.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md), [String](../../sql-reference/data-types/string.md)))) — настройки, которые были изменены при входе или выходе клиента из системы.
- `client_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP-адрес, который использовался для входа или выхода из системы.
- `client_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP-адрес, который использовался для входа или выхода из системы.
- `client_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт клиента, который использовался для входа или выхода из системы.
- `interface` ([Enum8](../../sql-reference/data-types/enum.md)) — интерфейс, с которого был инициирован вход в систему. Возможные значения:
- `TCP`

View File

@ -15,7 +15,7 @@ slug: /ru/operations/system-tables/zookeeper_log
- `Finalize` — соединение разорвано, ответ не получен.
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата, когда произошло событие.
- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — дата и время, когда произошло событие.
- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес сервера ZooKeeper, с которого был сделан запрос.
- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — IP адрес сервера ZooKeeper, с которого был сделан запрос.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт сервера ZooKeeper, с которого был сделан запрос.
- `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор сессии, который сервер ZooKeeper создает для каждого соединения.
- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — идентификатор запроса внутри сессии. Обычно это последовательный номер запроса, одинаковый у строки запроса и у парной строки `response`/`finalize`.

View File

@ -1,12 +1,12 @@
---
slug: /ru/sql-reference/data-types/domains/ipv4
slug: /ru/sql-reference/data-types/ipv4
sidebar_position: 59
sidebar_label: IPv4
---
## IPv4 {#ipv4}
`IPv4` — это домен, базирующийся на типе данных `UInt32` предназначенный для хранения адресов IPv4. Он обеспечивает компактное хранение данных с удобным для человека форматом ввода-вывода, и явно отображаемым типом данных в структуре таблицы.
IPv4-адреса. Хранится в 4 байтах как UInt32.
### Применение {#primenenie}
@ -57,27 +57,6 @@ SELECT toTypeName(from), hex(from) FROM hits LIMIT 1;
└──────────────────┴───────────┘
```
Значения с доменным типом данных не преобразуются неявно в другие типы данных, кроме `UInt32`.
Если необходимо преобразовать значение типа `IPv4` в строку, то это необходимо делать явно с помощью функции `IPv4NumToString()`:
**См. также**
``` sql
SELECT toTypeName(s), IPv4NumToString(from) AS s FROM hits LIMIT 1;
```
``` text
┌─toTypeName(IPv4NumToString(from))─┬─s──────────────┐
│ String │ 183.247.232.58 │
└───────────────────────────────────┴────────────────┘
```
Или приводить к типу данных `UInt32`:
``` sql
SELECT toTypeName(i), CAST(from AS UInt32) AS i FROM hits LIMIT 1;
```
``` text
┌─toTypeName(CAST(from, 'UInt32'))─┬──────────i─┐
│ UInt32 │ 3086477370 │
└──────────────────────────────────┴────────────┘
```
- [Functions for Working with IPv4 and IPv6 Addresses](../functions/ip-address-functions.md)

View File

@ -1,5 +1,5 @@
---
slug: /ru/sql-reference/data-types/domains/ipv6
slug: /ru/sql-reference/data-types/ipv6
sidebar_position: 60
sidebar_label: IPv6
---

View File

@ -26,7 +26,8 @@ SELECT
## timeZone {#timezone}
Возвращает часовой пояс сервера.
Возвращает часовой пояс сервера, считающийся умолчанием для текущей сессии: значение параметра [session_timezone](../../operations/settings/settings.md#session_timezone), если установлено.
Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями, актуальными для каждого шарда. Иначе возвращается константа.
**Синтаксис**
@ -43,6 +44,33 @@ timeZone()
Тип: [String](../../sql-reference/data-types/string.md).
**Смотрите также**
- [serverTimeZone](#servertimezone)
## serverTimeZone {#servertimezone}
Возвращает часовой пояс сервера по умолчанию, в т.ч. установленный [timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone)
Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями, актуальными для каждого шарда. Иначе возвращается константа.
**Синтаксис**
``` sql
serverTimeZone()
```
Синонимы: `serverTimezone`.
**Возвращаемое значение**
- Часовой пояс.
Тип: [String](../../sql-reference/data-types/string.md).
**Смотрите также**
- [timeZone](#timezone)
## toTimeZone {#totimezone}
Переводит дату или дату с временем в указанный часовой пояс. Часовой пояс - это атрибут типов `Date` и `DateTime`. Внутреннее значение (количество секунд) поля таблицы или результирующего столбца не изменяется, изменяется тип поля и, соответственно, его текстовое отображение.

View File

@ -5,7 +5,7 @@ sidebar_label: "Функции для работы с внешними слов
---
:::note "Внимание"
Для словарей, созданных с помощью [DDL-запросов](../../sql-reference/statements/create/dictionary.md), в параметре `dict_name` указывается полное имя словаря вместе с базой данных, например: `<database>.<dict_name>`. Если база данных не указана, используется текущая.
Для словарей, созданных с помощью [DDL-запросов](../../sql-reference/statements/create/dictionary.md), в параметре `dict_name` указывается полное имя словаря вместе с базой данных, например: `<database>.<dict_name>`. Если база данных не указана, используется текущая.
:::
# Функции для работы с внешними словарями {#ext_dict_functions}

View File

@ -265,7 +265,7 @@ SELECT
## toIPv6 {#toipv6string}
Приводит строку с адресом в формате IPv6 к типу [IPv6](../../sql-reference/data-types/domains/ipv6.md). Возвращает пустое значение, если входящая строка не является корректным IP адресом.
Приводит строку с адресом в формате IPv6 к типу [IPv6](../../sql-reference/data-types/ipv6.md). Возвращает пустое значение, если входящая строка не является корректным IP адресом.
Похоже на функцию [IPv6StringToNum](#ipv6stringtonums), которая представляет адрес IPv6 в двоичном виде.
Если входящая строка содержит корректный IPv4 адрес, функция возвращает его IPv6 эквивалент.
@ -284,7 +284,7 @@ toIPv6(string)
- IP адрес.
Тип: [IPv6](../../sql-reference/data-types/domains/ipv6.md).
Тип: [IPv6](../../sql-reference/data-types/ipv6.md).
**Примеры**

View File

@ -60,11 +60,11 @@ ClickHouse不会自动从表中删除数据。更多详情请看 [introduction](
- 0 — 由另一个查询发起的,作为分布式查询的一部分.
- `user` ([String](../../sql-reference/data-types/string.md)) — 发起查询的用户.
- `query_id` ([String](../../sql-reference/data-types/string.md)) — 查询ID.
- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 发起查询的客户端IP地址.
- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — 发起查询的客户端IP地址.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — 发起查询的客户端端口.
- `initial_user` ([String](../../sql-reference/data-types/string.md)) — 初始查询的用户名(用于分布式查询执行).
- `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — 运行初始查询的ID用于分布式查询执行.
- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 运行父查询的IP地址.
- `initial_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — 运行父查询的IP地址.
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — 发起父查询的客户端端口.
- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md)) — 发起查询的接口. 可能的值:
- 1 — TCP.

View File

@ -36,11 +36,11 @@ ClickHouse不会自动从表中删除数据。 欲了解更多详情,请参照
- 0 — 由其他查询发起的分布式查询。
- `user` ([字符串](../../sql-reference/data-types/string.md)) — 发起查询的用户名。
- `query_id` ([字符串](../../sql-reference/data-types/string.md)) — 查询的ID。
- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 发起查询的IP地址。
- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — 发起查询的IP地址。
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起查询的端口。
- `initial_user` ([字符串](../../sql-reference/data-types/string.md)) — 首次发起查询的用户名(对于分布式查询)。
- `initial_query_id` ([字符串](../../sql-reference/data-types/string.md)) — 首次发起查询的ID对于分布式查询
- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 发起该查询的父查询IP地址。
- `initial_address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — 发起该查询的父查询IP地址。
- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起该查询的父查询端口。
- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起查询的界面,可能的值:
- 1 — TCP.

View File

@ -15,7 +15,7 @@ slug: /zh/operations/system-tables/zookeeper_log
- `Finalize` — 连接丢失, 未收到响应.
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — 事件发生的日期.
- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — 事件发生的日期和时间.
- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 用于发出请求的 ZooKeeper 服务器的 IP 地址.
- `address` ([IPv6](../../sql-reference/data-types/ipv6.md)) — 用于发出请求的 ZooKeeper 服务器的 IP 地址.
- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — 用于发出请求的 ZooKeeper 服务器的端口.
- `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — ZooKeeper 服务器为每个连接设置的会话 ID.
- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — 会话中请求的 ID. 这通常是一个连续的请求编号. 请求行和配对的 `response`/`finalize` 行相同.

View File

@ -1,5 +1,5 @@
---
slug: /zh/sql-reference/data-types/domains/ipv4
slug: /zh/sql-reference/data-types/ipv4
---
## IPv4 {#ipv4}

View File

@ -1,5 +1,5 @@
---
slug: /zh/sql-reference/data-types/domains/ipv6
slug: /zh/sql-reference/data-types/ipv6
---
## IPv6 {#ipv6}

View File

@ -4,7 +4,9 @@
#include <map>
#include <iostream>
#include <iomanip>
#include <memory>
#include <optional>
#include <Common/ThreadStatus.h>
#include <Common/scope_guard_safe.h>
#include <boost/program_options.hpp>
#include <boost/algorithm/string/replace.hpp>
@ -307,7 +309,7 @@ int Client::main(const std::vector<std::string> & /*args*/)
try
{
UseSSL use_ssl;
MainThreadStatus::getInstance();
auto & thread_status = MainThreadStatus::getInstance();
setupSignalHandler();
std::cout << std::fixed << std::setprecision(3);
@ -320,6 +322,14 @@ try
processConfig();
initTtyBuffer(toProgressOption(config().getString("progress", "default")));
{
// All that just to set DB::CurrentThread::get().getGlobalContext()
// which is required for client timezone (pushed from server) to work.
auto thread_group = std::make_shared<ThreadGroup>();
const_cast<ContextWeakPtr&>(thread_group->global_context) = global_context;
thread_status.attachToGroup(thread_group, false);
}
/// Includes delayed_interactive.
if (is_interactive)
{

View File

@ -44,7 +44,7 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self)
time_t timestamp = Poco::Timestamp().epochTime();
auto curr_pid = Poco::Process::id();
process_id = std::to_string(DateLUT::instance().toNumYYYYMMDDhhmmss(timestamp)) + "_" + std::to_string(curr_pid);
process_id = std::to_string(DateLUT::serverTimezoneInstance().toNumYYYYMMDDhhmmss(timestamp)) + "_" + std::to_string(curr_pid);
host_id = escapeForFileName(getFQDNOrHostName()) + '#' + process_id;
process_path = fs::weakly_canonical(fs::path(base_dir) / ("clickhouse-copier_" + process_id));
fs::create_directories(process_path);

View File

@ -224,12 +224,12 @@ struct Keeper::KeeperHTTPContext : public IHTTPContext
uint64_t getMaxFieldNameSize() const override
{
return context->getConfigRef().getUInt64("keeper_server.http_max_field_name_size", 1048576);
return context->getConfigRef().getUInt64("keeper_server.http_max_field_name_size", 128 * 1024);
}
uint64_t getMaxFieldValueSize() const override
{
return context->getConfigRef().getUInt64("keeper_server.http_max_field_value_size", 1048576);
return context->getConfigRef().getUInt64("keeper_server.http_max_field_value_size", 128 * 1024);
}
uint64_t getMaxChunkSize() const override
@ -306,8 +306,8 @@ try
/// Initialize DateLUT early, to not interfere with running time of first query.
LOG_DEBUG(log, "Initializing DateLUT.");
DateLUT::instance();
LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::instance().getTimeZone());
DateLUT::serverTimezoneInstance();
LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::serverTimezoneInstance().getTimeZone());
/// Don't want to use DNS cache
DNSResolver::instance().setDisableCacheFlag();

View File

@ -491,7 +491,7 @@ private:
const DateLUTImpl & date_lut;
public:
explicit DateTimeModel(UInt64 seed_) : seed(seed_), date_lut(DateLUT::instance()) {}
explicit DateTimeModel(UInt64 seed_) : seed(seed_), date_lut(DateLUT::serverTimezoneInstance()) {}
void train(const IColumn &) override {}
void finalize() override {}

View File

@ -960,8 +960,8 @@ try
/// Initialize DateLUT early, to not interfere with running time of first query.
LOG_DEBUG(log, "Initializing DateLUT.");
DateLUT::instance();
LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::instance().getTimeZone());
DateLUT::serverTimezoneInstance();
LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::serverTimezoneInstance().getTimeZone());
/// Storage with temporary data for processing of heavy queries.
if (!server_settings.tmp_policy.value.empty())

View File

@ -1543,12 +1543,12 @@
-->
<!-- Configuration for the query cache -->
<!-- <query_cache> -->
<!-- <max_size_in_bytes>1073741824</max_size_in_bytes> -->
<!-- <max_entries>1024</max_entries> -->
<!-- <max_entry_size_in_bytes>1048576</max_entry_size_in_bytes> -->
<!-- <max_entry_size_in_rows>30000000</max_entry_size_in_rows> -->
<!-- </query_cache> -->
<query_cache>
<max_size_in_bytes>1073741824</max_size_in_bytes>
<max_entries>1024</max_entries>
<max_entry_size_in_bytes>1048576</max_entry_size_in_bytes>
<max_entry_size_in_rows>30000000</max_entry_size_in_rows>
</query_cache>
<!-- Uncomment if enable merge tree metadata cache -->
<!--merge_tree_metadata_cache>

View File

@ -78,14 +78,14 @@ namespace detail
void serialize(WriteBuffer & buf) const
{
writeBinary(count, buf);
writeBinaryLittleEndian(count, buf);
buf.write(reinterpret_cast<const char *>(elems), count * sizeof(elems[0]));
}
void deserialize(ReadBuffer & buf)
{
UInt16 new_count = 0;
readBinary(new_count, buf);
readBinaryLittleEndian(new_count, buf);
if (new_count > TINY_MAX_ELEMS)
throw Exception(ErrorCodes::INCORRECT_DATA, "The number of elements {} for the 'tiny' kind of quantileTiming is exceeding the maximum of {}", new_count, TINY_MAX_ELEMS);
buf.readStrict(reinterpret_cast<char *>(elems), new_count * sizeof(elems[0]));
@ -164,14 +164,14 @@ namespace detail
void serialize(WriteBuffer & buf) const
{
writeBinary(elems.size(), buf);
writeBinaryLittleEndian(elems.size(), buf);
buf.write(reinterpret_cast<const char *>(elems.data()), elems.size() * sizeof(elems[0]));
}
void deserialize(ReadBuffer & buf)
{
size_t size = 0;
readBinary(size, buf);
readBinaryLittleEndian(size, buf);
if (size > 10'000)
throw Exception(ErrorCodes::INCORRECT_DATA, "The number of elements {} for the 'medium' kind of quantileTiming is too large", size);
@ -341,7 +341,7 @@ namespace detail
void serialize(WriteBuffer & buf) const
{
writeBinary(count, buf);
writeBinaryLittleEndian(count, buf);
if (count * 2 > SMALL_THRESHOLD + BIG_SIZE)
{
@ -356,8 +356,8 @@ namespace detail
{
if (count_small[i])
{
writeBinary(UInt16(i), buf);
writeBinary(count_small[i], buf);
writeBinaryLittleEndian(UInt16(i), buf);
writeBinaryLittleEndian(count_small[i], buf);
}
}
@ -365,19 +365,19 @@ namespace detail
{
if (count_big[i])
{
writeBinary(UInt16(i + SMALL_THRESHOLD), buf);
writeBinary(count_big[i], buf);
writeBinaryLittleEndian(UInt16(i + SMALL_THRESHOLD), buf);
writeBinaryLittleEndian(count_big[i], buf);
}
}
/// Symbolizes end of data.
writeBinary(UInt16(BIG_THRESHOLD), buf);
writeBinaryLittleEndian(UInt16(BIG_THRESHOLD), buf);
}
}
void deserialize(ReadBuffer & buf)
{
readBinary(count, buf);
readBinaryLittleEndian(count, buf);
if (count * 2 > SMALL_THRESHOLD + BIG_SIZE)
{
@ -388,12 +388,12 @@ namespace detail
while (true)
{
UInt16 index = 0;
readBinary(index, buf);
readBinaryLittleEndian(index, buf);
if (index == BIG_THRESHOLD)
break;
UInt64 elem_count = 0;
readBinary(elem_count, buf);
readBinaryLittleEndian(elem_count, buf);
if (index < SMALL_THRESHOLD)
count_small[index] = elem_count;
@ -692,7 +692,7 @@ public:
void serialize(WriteBuffer & buf) const
{
auto kind = which();
DB::writePODBinary(kind, buf);
writeBinaryLittleEndian(kind, buf);
if (kind == Kind::Tiny)
tiny.serialize(buf);
@ -706,7 +706,7 @@ public:
void deserialize(ReadBuffer & buf)
{
Kind kind;
DB::readPODBinary(kind, buf);
readBinaryLittleEndian(kind, buf);
if (kind == Kind::Tiny)
{

View File

@ -721,7 +721,15 @@ void BackupCoordinationRemote::prepareFileInfos() const
bool BackupCoordinationRemote::startWritingFile(size_t data_file_index)
{
bool acquired_writing = false;
{
/// Check if this host is already writing this file.
std::lock_guard lock{writing_files_mutex};
if (writing_files.contains(data_file_index))
return false;
}
/// Store in Zookeeper that this host is the only host which is allowed to write this file.
bool host_is_assigned = false;
String full_path = zookeeper_path + "/writing_files/" + std::to_string(data_file_index);
String host_index_str = std::to_string(current_host_index);
@ -733,14 +741,23 @@ bool BackupCoordinationRemote::startWritingFile(size_t data_file_index)
auto code = zk->tryCreate(full_path, host_index_str, zkutil::CreateMode::Persistent);
if (code == Coordination::Error::ZOK)
acquired_writing = true; /// If we've just created this ZooKeeper's node, the writing is acquired, i.e. we should write this data file.
host_is_assigned = true; /// If we've just created this ZooKeeper's node, this host is assigned.
else if (code == Coordination::Error::ZNODEEXISTS)
acquired_writing = (zk->get(full_path) == host_index_str); /// The previous retry could write this ZooKeeper's node and then fail.
host_is_assigned = (zk->get(full_path) == host_index_str); /// The previous retry could write this ZooKeeper's node and then fail.
else
throw zkutil::KeeperException(code, full_path);
});
return acquired_writing;
if (!host_is_assigned)
return false; /// Other host is writing this file.
{
/// Check if this host is already writing this file,
/// and if it's not, mark that this host is writing this file.
/// We have to check that again because we were accessing ZooKeeper with the mutex unlocked.
std::lock_guard lock{writing_files_mutex};
return writing_files.emplace(data_file_index).second; /// Return false if this host is already writing this file.
}
}
bool BackupCoordinationRemote::hasConcurrentBackups(const std::atomic<size_t> &) const

View File

@ -106,12 +106,14 @@ private:
mutable std::optional<BackupCoordinationReplicatedAccess> TSA_GUARDED_BY(replicated_access_mutex) replicated_access;
mutable std::optional<BackupCoordinationReplicatedSQLObjects> TSA_GUARDED_BY(replicated_sql_objects_mutex) replicated_sql_objects;
mutable std::optional<BackupCoordinationFileInfos> TSA_GUARDED_BY(file_infos_mutex) file_infos;
std::unordered_set<size_t> TSA_GUARDED_BY(writing_files_mutex) writing_files;
mutable std::mutex zookeeper_mutex;
mutable std::mutex replicated_tables_mutex;
mutable std::mutex replicated_access_mutex;
mutable std::mutex replicated_sql_objects_mutex;
mutable std::mutex file_infos_mutex;
mutable std::mutex writing_files_mutex;
};
}

View File

@ -165,7 +165,7 @@ void IBridge::initialize(Application & self)
http_timeout = config().getUInt64("http-timeout", DEFAULT_HTTP_READ_BUFFER_TIMEOUT);
max_server_connections = config().getUInt("max-server-connections", 1024);
keep_alive_timeout = config().getUInt64("keep-alive-timeout", 10);
http_max_field_value_size = config().getUInt64("http-max-field-value-size", 1048576);
http_max_field_value_size = config().getUInt64("http-max-field-value-size", 128 * 1024);
struct rlimit limit;
const UInt64 gb = 1024 * 1024 * 1024;

View File

@ -77,7 +77,6 @@
#include "config_version.h"
#include "config.h"
namespace fs = std::filesystem;
using namespace std::literals;
@ -896,7 +895,6 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa
if (send_external_tables)
sendExternalTables(parsed_query);
receiveResult(parsed_query, signals_before_stop, settings.partial_result_on_first_cancel);
break;
@ -1048,6 +1046,10 @@ bool ClientBase::receiveAndProcessPacket(ASTPtr parsed_query, bool cancelled_)
onProfileEvents(packet.block);
return true;
case Protocol::Server::TimezoneUpdate:
onTimezoneUpdate(packet.server_timezone);
return true;
default:
throw Exception(
ErrorCodes::UNKNOWN_PACKET_FROM_SERVER, "Unknown packet {} from server {}", packet.type, connection->getDescription());
@ -1070,6 +1072,11 @@ void ClientBase::onProgress(const Progress & value)
progress_indication.writeProgress(*tty_buf);
}
void ClientBase::onTimezoneUpdate(const String & tz)
{
global_context->setSetting("session_timezone", tz);
}
void ClientBase::onEndOfStream()
{
@ -1221,9 +1228,13 @@ bool ClientBase::receiveSampleBlock(Block & out, ColumnsDescription & columns_de
columns_description = ColumnsDescription::parse(packet.multistring_message[1]);
return receiveSampleBlock(out, columns_description, parsed_query);
case Protocol::Server::TimezoneUpdate:
onTimezoneUpdate(packet.server_timezone);
break;
default:
throw NetException(ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER,
"Unexpected packet from server (expected Data, Exception or Log, got {})",
"Unexpected packet from server (expected Data, Exception, Log or TimezoneUpdate, got {})",
String(Protocol::Server::toString(packet.type)));
}
}
@ -1538,7 +1549,9 @@ void ClientBase::receiveLogsAndProfileEvents(ASTPtr parsed_query)
{
auto packet_type = connection->checkPacket(0);
while (packet_type && (*packet_type == Protocol::Server::Log || *packet_type == Protocol::Server::ProfileEvents))
while (packet_type && (*packet_type == Protocol::Server::Log
|| *packet_type == Protocol::Server::ProfileEvents
|| *packet_type == Protocol::Server::TimezoneUpdate))
{
receiveAndProcessPacket(parsed_query, false);
packet_type = connection->checkPacket(0);
@ -1575,6 +1588,10 @@ bool ClientBase::receiveEndOfQuery()
onProfileEvents(packet.block);
break;
case Protocol::Server::TimezoneUpdate:
onTimezoneUpdate(packet.server_timezone);
break;
default:
throw NetException(ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER,
"Unexpected packet from server (expected Exception, EndOfStream, Log, Progress or ProfileEvents. Got {})",

View File

@ -148,6 +148,7 @@ private:
void cancelQuery();
void onProgress(const Progress & value);
void onTimezoneUpdate(const String & tz);
void onData(Block & block, ASTPtr parsed_query);
void onLogData(Block & block);
void onTotals(Block & block, ASTPtr parsed_query);

View File

@ -1022,6 +1022,11 @@ Packet Connection::receivePacket()
res.block = receiveProfileEvents();
return res;
case Protocol::Server::TimezoneUpdate:
readStringBinary(server_timezone, *in);
res.server_timezone = server_timezone;
return res;
default:
/// In unknown state, disconnect - to not leave unsynchronised connection.
disconnect();

View File

@ -419,6 +419,7 @@ Packet HedgedConnections::receivePacketFromReplica(const ReplicaLocation & repli
}
replica_with_last_received_packet = replica_location;
break;
case Protocol::Server::TimezoneUpdate:
case Protocol::Server::PartUUIDs:
case Protocol::Server::ProfileInfo:
case Protocol::Server::Totals:

View File

@ -38,6 +38,8 @@ struct Packet
ParallelReadRequest request;
ParallelReadResponse response;
std::string server_timezone;
Packet() : type(Protocol::Server::Hello) {}
};

View File

@ -259,6 +259,7 @@ Packet MultiplexedConnections::drain()
switch (packet.type)
{
case Protocol::Server::TimezoneUpdate:
case Protocol::Server::MergeTreeAllRangesAnnounecement:
case Protocol::Server::MergeTreeReadTaskRequest:
case Protocol::Server::ReadTaskRequest:
@ -340,6 +341,7 @@ Packet MultiplexedConnections::receivePacketUnlocked(AsyncCallback async_callbac
switch (packet.type)
{
case Protocol::Server::TimezoneUpdate:
case Protocol::Server::MergeTreeAllRangesAnnounecement:
case Protocol::Server::MergeTreeReadTaskRequest:
case Protocol::Server::ReadTaskRequest:

View File

@ -160,6 +160,7 @@ void Suggest::fetch(IServerConnection & connection, const ConnectionTimeouts & t
fillWordsFromBlock(packet.block);
continue;
case Protocol::Server::TimezoneUpdate:
case Protocol::Server::Progress:
case Protocol::Server::ProfileInfo:
case Protocol::Server::Totals:

View File

@ -200,9 +200,11 @@ void AsyncLoader::start()
void AsyncLoader::wait()
{
// Because job can create new jobs in other pools we have to recheck in cycle
// Because job can create new jobs in other pools we have to recheck in cycle.
// Also wait for all workers to finish to avoid races on `pool.workers`,
// which can decrease even after all jobs are already finished.
std::unique_lock lock{mutex};
while (!scheduled_jobs.empty())
while (!scheduled_jobs.empty() || hasWorker(lock))
{
lock.unlock();
for (auto & p : pools)
@ -719,4 +721,14 @@ void AsyncLoader::worker(Pool & pool)
}
}
bool AsyncLoader::hasWorker(std::unique_lock<std::mutex> &) const
{
for (const Pool & pool : pools)
{
if (pool.workers > 0)
return true;
}
return false;
}
}

View File

@ -445,6 +445,7 @@ private:
void updateCurrentPriorityAndSpawn(std::unique_lock<std::mutex> &);
void spawn(Pool & pool, std::unique_lock<std::mutex> &);
void worker(Pool & pool);
bool hasWorker(std::unique_lock<std::mutex> &) const;
// Logging
const bool log_failures; // Worker should log all exceptions caught from job functions.

View File

@ -7,6 +7,7 @@
#include <filesystem>
#include <fstream>
#include <Interpreters/Context.h>
namespace
@ -163,3 +164,8 @@ DateLUT & DateLUT::getInstance()
static DateLUT ret;
return ret;
}
std::string DateLUT::extractTimezoneFromContext(DB::ContextPtr query_context)
{
return query_context->getSettingsRef().session_timezone.value;
}

View File

@ -5,6 +5,7 @@
#include <base/defines.h>
#include <boost/noncopyable.hpp>
#include "Common/CurrentThread.h"
#include <atomic>
#include <memory>
@ -16,22 +17,59 @@
class DateLUT : private boost::noncopyable
{
public:
/// Return singleton DateLUTImpl instance for the default time zone.
/// Return DateLUTImpl instance for session timezone.
/// session_timezone is a session-level setting.
/// If setting is not set, returns the server timezone.
static ALWAYS_INLINE const DateLUTImpl & instance()
{
const auto & date_lut = getInstance();
if (DB::CurrentThread::isInitialized())
{
std::string timezone_from_context;
const DB::ContextPtr query_context = DB::CurrentThread::get().getQueryContext();
if (query_context)
{
timezone_from_context = extractTimezoneFromContext(query_context);
if (!timezone_from_context.empty())
return date_lut.getImplementation(timezone_from_context);
}
/// On the server side, timezone is passed in query_context,
/// but on CH-client side we have no query context,
/// and each time we modify client's global context
const DB::ContextPtr global_context = DB::CurrentThread::get().getGlobalContext();
if (global_context)
{
timezone_from_context = extractTimezoneFromContext(global_context);
if (!timezone_from_context.empty())
return date_lut.getImplementation(timezone_from_context);
}
}
return serverTimezoneInstance();
}
static ALWAYS_INLINE const DateLUTImpl & instance(const std::string & time_zone)
{
if (time_zone.empty())
return instance();
const auto & date_lut = getInstance();
return date_lut.getImplementation(time_zone);
}
/// Return singleton DateLUTImpl for the server time zone.
/// It may be set using 'timezone' server setting.
static ALWAYS_INLINE const DateLUTImpl & serverTimezoneInstance()
{
const auto & date_lut = getInstance();
return *date_lut.default_impl.load(std::memory_order_acquire);
}
/// Return singleton DateLUTImpl instance for a given time zone.
static ALWAYS_INLINE const DateLUTImpl & instance(const std::string & time_zone)
{
const auto & date_lut = getInstance();
if (time_zone.empty())
return *date_lut.default_impl.load(std::memory_order_acquire);
return date_lut.getImplementation(time_zone);
}
static void setDefaultTimezone(const std::string & time_zone)
{
auto & date_lut = getInstance();
@ -45,6 +83,8 @@ protected:
private:
static DateLUT & getInstance();
static std::string extractTimezoneFromContext(DB::ContextPtr query_context);
const DateLUTImpl & getImplementation(const std::string & time_zone) const;
using DateLUTImplPtr = std::unique_ptr<DateLUTImpl>;

View File

@ -24,9 +24,8 @@ private:
unsigned char m_month;
unsigned char m_day;
void init(time_t time)
void init(time_t time, const DateLUTImpl & date_lut)
{
const auto & date_lut = DateLUT::instance();
const auto & values = date_lut.getValues(time);
m_year = values.year;
@ -56,22 +55,22 @@ private:
}
public:
explicit LocalDate(time_t time)
explicit LocalDate(time_t time, const DateLUTImpl & time_zone = DateLUT::instance())
{
init(time);
init(time, time_zone);
}
LocalDate(DayNum day_num) /// NOLINT
LocalDate(DayNum day_num, const DateLUTImpl & time_zone = DateLUT::instance()) /// NOLINT
{
const auto & values = DateLUT::instance().getValues(day_num);
const auto & values = time_zone.getValues(day_num);
m_year = values.year;
m_month = values.month;
m_day = values.day_of_month;
}
explicit LocalDate(ExtendedDayNum day_num)
explicit LocalDate(ExtendedDayNum day_num, const DateLUTImpl & time_zone = DateLUT::instance())
{
const auto & values = DateLUT::instance().getValues(day_num);
const auto & values = time_zone.getValues(day_num);
m_year = values.year;
m_month = values.month;
m_day = values.day_of_month;
@ -99,15 +98,13 @@ public:
LocalDate(const LocalDate &) noexcept = default;
LocalDate & operator= (const LocalDate &) noexcept = default;
DayNum getDayNum() const
DayNum getDayNum(const DateLUTImpl & lut = DateLUT::instance()) const
{
const auto & lut = DateLUT::instance();
return DayNum(lut.makeDayNum(m_year, m_month, m_day).toUnderType());
}
ExtendedDayNum getExtenedDayNum() const
ExtendedDayNum getExtenedDayNum(const DateLUTImpl & lut = DateLUT::instance()) const
{
const auto & lut = DateLUT::instance();
return ExtendedDayNum (lut.makeDayNum(m_year, m_month, m_day).toUnderType());
}

View File

@ -1107,17 +1107,20 @@ void ZooKeeper::initApiVersion()
get(keeper_api_version_path, std::move(callback), {});
if (future.wait_for(std::chrono::milliseconds(args.operation_timeout_ms)) != std::future_status::ready)
{
LOG_TRACE(log, "Failed to get API version: timeout");
return;
throw Exception(Error::ZOPERATIONTIMEOUT, "Failed to get API version: timeout");
}
auto response = future.get();
if (response.error != Coordination::Error::ZOK)
if (response.error == Coordination::Error::ZNONODE)
{
LOG_TRACE(log, "Failed to get API version");
LOG_TRACE(log, "API version not found, assuming {}", keeper_api_version);
return;
}
else if (response.error != Coordination::Error::ZOK)
{
throw Exception(response.error, "Failed to get API version");
}
uint8_t keeper_version{0};
DB::ReadBufferFromOwnString buf(response.data);

View File

@ -83,7 +83,8 @@ namespace Protocol
ProfileEvents = 14, /// Packet with profile events from server.
MergeTreeAllRangesAnnounecement = 15,
MergeTreeReadTaskRequest = 16, /// Request from a MergeTree replica to a coordinator
MAX = MergeTreeReadTaskRequest,
TimezoneUpdate = 17, /// Receive server's (session-wide) default timezone
MAX = TimezoneUpdate,
};
@ -111,6 +112,7 @@ namespace Protocol
"ProfileEvents",
"MergeTreeAllRangesAnnounecement",
"MergeTreeReadTaskRequest",
"TimezoneUpdate",
};
return packet <= MAX
? data[packet]

View File

@ -53,7 +53,7 @@
/// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION,
/// later is just a number for server version (one number instead of commit SHA)
/// for simplicity (sometimes it may be more convenient in some use cases).
#define DBMS_TCP_PROTOCOL_VERSION 54463
#define DBMS_TCP_PROTOCOL_VERSION 54464
#define DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME 54449
@ -75,3 +75,5 @@
#define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 54462
#define DBMS_MIN_PROTOCOL_VERSION_WITH_TOTAL_BYTES_IN_PROGRESS 54463
#define DBMS_MIN_PROTOCOL_VERSION_WITH_TIMEZONE_UPDATES 54464

View File

@ -315,8 +315,8 @@ class IColumn;
M(Seconds, http_receive_timeout, DEFAULT_HTTP_READ_BUFFER_TIMEOUT, "HTTP receive timeout", 0) \
M(UInt64, http_max_uri_size, 1048576, "Maximum URI length of HTTP request", 0) \
M(UInt64, http_max_fields, 1000000, "Maximum number of fields in HTTP header", 0) \
M(UInt64, http_max_field_name_size, 1048576, "Maximum length of field name in HTTP header", 0) \
M(UInt64, http_max_field_value_size, 1048576, "Maximum length of field value in HTTP header", 0) \
M(UInt64, http_max_field_name_size, 128 * 1024, "Maximum length of field name in HTTP header", 0) \
M(UInt64, http_max_field_value_size, 128 * 1024, "Maximum length of field value in HTTP header", 0) \
M(UInt64, http_max_chunk_size, 100_GiB, "Maximum value of a chunk size in HTTP chunked transfer encoding", 0) \
M(Bool, http_skip_not_found_url_for_globs, true, "Skip url's for globs with HTTP_NOT_FOUND error", 0) \
M(Bool, optimize_throw_if_noop, false, "If setting is enabled and OPTIMIZE query didn't actually assign a merge then an explanatory exception is thrown", 0) \
@ -770,6 +770,7 @@ class IColumn;
M(Bool, allow_experimental_undrop_table_query, false, "Allow to use undrop query to restore dropped table in a limited time", 0) \
M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \
M(UInt64, extract_kvp_max_pairs_per_row, 1000, "Max number pairs that can be produced by extractKeyValuePairs function. Used to safeguard against consuming too much memory.", 0) \
M(Timezone, session_timezone, "", "The default timezone for current session or query. The server default timezone if empty.", 0) \
// End of COMMON_SETTINGS
// Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS.

View File

@ -1,8 +1,10 @@
#include <Core/SettingsEnums.h>
#include <magic_enum.hpp>
namespace DB
{
namespace ErrorCodes
{
extern const int UNKNOWN_LOAD_BALANCING;
@ -15,7 +17,6 @@ namespace ErrorCodes
extern const int UNKNOWN_UNION;
}
IMPLEMENT_SETTING_ENUM(LoadBalancing, ErrorCodes::UNKNOWN_LOAD_BALANCING,
{{"random", LoadBalancing::RANDOM},
{"nearest_hostname", LoadBalancing::NEAREST_HOSTNAME},
@ -54,7 +55,7 @@ IMPLEMENT_SETTING_ENUM(OverflowMode, ErrorCodes::UNKNOWN_OVERFLOW_MODE,
{"break", OverflowMode::BREAK}})
IMPLEMENT_SETTING_ENUM_WITH_RENAME(OverflowModeGroupBy, ErrorCodes::UNKNOWN_OVERFLOW_MODE,
IMPLEMENT_SETTING_ENUM(OverflowModeGroupBy, ErrorCodes::UNKNOWN_OVERFLOW_MODE,
{{"throw", OverflowMode::THROW},
{"break", OverflowMode::BREAK},
{"any", OverflowMode::ANY}})
@ -67,51 +68,26 @@ IMPLEMENT_SETTING_ENUM(DistributedProductMode, ErrorCodes::UNKNOWN_DISTRIBUTED_P
{"allow", DistributedProductMode::ALLOW}})
IMPLEMENT_SETTING_ENUM_WITH_RENAME(DateTimeInputFormat, ErrorCodes::BAD_ARGUMENTS,
IMPLEMENT_SETTING_ENUM(DateTimeInputFormat, ErrorCodes::BAD_ARGUMENTS,
{{"basic", FormatSettings::DateTimeInputFormat::Basic},
{"best_effort", FormatSettings::DateTimeInputFormat::BestEffort},
{"best_effort_us", FormatSettings::DateTimeInputFormat::BestEffortUS}})
IMPLEMENT_SETTING_ENUM_WITH_RENAME(DateTimeOutputFormat, ErrorCodes::BAD_ARGUMENTS,
IMPLEMENT_SETTING_ENUM(DateTimeOutputFormat, ErrorCodes::BAD_ARGUMENTS,
{{"simple", FormatSettings::DateTimeOutputFormat::Simple},
{"iso", FormatSettings::DateTimeOutputFormat::ISO},
{"unix_timestamp", FormatSettings::DateTimeOutputFormat::UnixTimestamp}})
IMPLEMENT_SETTING_ENUM(LogsLevel, ErrorCodes::BAD_ARGUMENTS,
{{"none", LogsLevel::none},
{"fatal", LogsLevel::fatal},
{"error", LogsLevel::error},
{"warning", LogsLevel::warning},
{"information", LogsLevel::information},
{"debug", LogsLevel::debug},
{"trace", LogsLevel::trace},
{"test", LogsLevel::test}})
IMPLEMENT_SETTING_AUTO_ENUM(LogsLevel, ErrorCodes::BAD_ARGUMENTS)
IMPLEMENT_SETTING_ENUM_WITH_RENAME(LogQueriesType, ErrorCodes::BAD_ARGUMENTS,
{{"QUERY_START", QUERY_START},
{"QUERY_FINISH", QUERY_FINISH},
{"EXCEPTION_BEFORE_START", EXCEPTION_BEFORE_START},
{"EXCEPTION_WHILE_PROCESSING", EXCEPTION_WHILE_PROCESSING}})
IMPLEMENT_SETTING_AUTO_ENUM(LogQueriesType, ErrorCodes::BAD_ARGUMENTS)
IMPLEMENT_SETTING_AUTO_ENUM(DefaultDatabaseEngine, ErrorCodes::BAD_ARGUMENTS)
IMPLEMENT_SETTING_ENUM_WITH_RENAME(DefaultDatabaseEngine, ErrorCodes::BAD_ARGUMENTS,
{{"Ordinary", DefaultDatabaseEngine::Ordinary},
{"Atomic", DefaultDatabaseEngine::Atomic}})
IMPLEMENT_SETTING_AUTO_ENUM(DefaultTableEngine, ErrorCodes::BAD_ARGUMENTS)
IMPLEMENT_SETTING_ENUM_WITH_RENAME(DefaultTableEngine, ErrorCodes::BAD_ARGUMENTS,
{{"None", DefaultTableEngine::None},
{"Log", DefaultTableEngine::Log},
{"StripeLog", DefaultTableEngine::StripeLog},
{"MergeTree", DefaultTableEngine::MergeTree},
{"ReplacingMergeTree", DefaultTableEngine::ReplacingMergeTree},
{"ReplicatedMergeTree", DefaultTableEngine::ReplicatedMergeTree},
{"ReplicatedReplacingMergeTree", DefaultTableEngine::ReplicatedReplacingMergeTree},
{"Memory", DefaultTableEngine::Memory}})
IMPLEMENT_SETTING_ENUM(CleanDeletedRows, ErrorCodes::BAD_ARGUMENTS,
{{"Never", CleanDeletedRows::Never},
{"Always", CleanDeletedRows::Always}})
IMPLEMENT_SETTING_AUTO_ENUM(CleanDeletedRows, ErrorCodes::BAD_ARGUMENTS)
IMPLEMENT_SETTING_MULTI_ENUM(MySQLDataTypesSupport, ErrorCodes::UNKNOWN_MYSQL_DATATYPES_SUPPORT_LEVEL,
{{"decimal", MySQLDataTypesSupport::DECIMAL},
@ -149,14 +125,7 @@ IMPLEMENT_SETTING_ENUM(CapnProtoEnumComparingMode, ErrorCodes::BAD_ARGUMENTS,
{"by_values", FormatSettings::CapnProtoEnumComparingMode::BY_VALUES},
{"by_names_case_insensitive", FormatSettings::CapnProtoEnumComparingMode::BY_NAMES_CASE_INSENSITIVE}})
IMPLEMENT_SETTING_ENUM(EscapingRule, ErrorCodes::BAD_ARGUMENTS,
{{"None", FormatSettings::EscapingRule::None},
{"Escaped", FormatSettings::EscapingRule::Escaped},
{"Quoted", FormatSettings::EscapingRule::Quoted},
{"CSV", FormatSettings::EscapingRule::CSV},
{"JSON", FormatSettings::EscapingRule::JSON},
{"XML", FormatSettings::EscapingRule::XML},
{"Raw", FormatSettings::EscapingRule::Raw}})
IMPLEMENT_SETTING_AUTO_ENUM(EscapingRule, ErrorCodes::BAD_ARGUMENTS)
IMPLEMENT_SETTING_ENUM(MsgPackUUIDRepresentation, ErrorCodes::BAD_ARGUMENTS,
{{"bin", FormatSettings::MsgPackUUIDRepresentation::BIN},
@ -166,17 +135,15 @@ IMPLEMENT_SETTING_ENUM(MsgPackUUIDRepresentation, ErrorCodes::BAD_ARGUMENTS,
IMPLEMENT_SETTING_ENUM(Dialect, ErrorCodes::BAD_ARGUMENTS,
{{"clickhouse", Dialect::clickhouse},
{"kusto", Dialect::kusto}})
// FIXME: do not add 'kusto_auto' to the list. Maybe remove it from code completely?
IMPLEMENT_SETTING_ENUM(ParallelReplicasCustomKeyFilterType, ErrorCodes::BAD_ARGUMENTS,
{{"default", ParallelReplicasCustomKeyFilterType::DEFAULT},
{"range", ParallelReplicasCustomKeyFilterType::RANGE}})
IMPLEMENT_SETTING_ENUM(LocalFSReadMethod, ErrorCodes::BAD_ARGUMENTS,
{{"mmap", LocalFSReadMethod::mmap},
{"pread", LocalFSReadMethod::pread},
{"read", LocalFSReadMethod::read}})
IMPLEMENT_SETTING_AUTO_ENUM(LocalFSReadMethod, ErrorCodes::BAD_ARGUMENTS)
IMPLEMENT_SETTING_ENUM_WITH_RENAME(ParquetVersion, ErrorCodes::BAD_ARGUMENTS,
IMPLEMENT_SETTING_ENUM(ParquetVersion, ErrorCodes::BAD_ARGUMENTS,
{{"1.0", FormatSettings::ParquetVersion::V1_0},
{"2.4", FormatSettings::ParquetVersion::V2_4},
{"2.6", FormatSettings::ParquetVersion::V2_6},

View File

@ -13,7 +13,6 @@
#include <cmath>
namespace DB
{
namespace ErrorCodes
@ -451,6 +450,17 @@ String SettingFieldEnumHelpers::readBinary(ReadBuffer & in)
return str;
}
void SettingFieldTimezone::writeBinary(WriteBuffer & out) const
{
writeStringBinary(value, out);
}
void SettingFieldTimezone::readBinary(ReadBuffer & in)
{
String str;
readStringBinary(str, in);
*this = std::move(str);
}
String SettingFieldCustom::toString() const
{

View File

@ -6,11 +6,11 @@
#include <Core/Field.h>
#include <Core/MultiEnum.h>
#include <boost/range/adaptor/map.hpp>
#include <cctz/time_zone.h>
#include <chrono>
#include <unordered_map>
#include <string_view>
namespace DB
{
namespace ErrorCodes
@ -371,19 +371,26 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
*this = Traits::fromString(SettingFieldEnumHelpers::readBinary(in));
}
template <typename Type>
constexpr auto getEnumValues()
{
std::array<std::pair<std::string_view, Type>, magic_enum::enum_count<Type>()> enum_values{};
size_t index = 0;
for (auto value : magic_enum::enum_values<Type>())
enum_values[index++] = std::pair{magic_enum::enum_name(value), value};
return enum_values;
}
/// NOLINTNEXTLINE
#define DECLARE_SETTING_ENUM(ENUM_TYPE) \
DECLARE_SETTING_ENUM_WITH_RENAME(ENUM_TYPE, ENUM_TYPE)
/// NOLINTNEXTLINE
#define IMPLEMENT_SETTING_ENUM(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \
IMPLEMENT_SETTING_ENUM_WITH_RENAME(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__)
/// NOLINTNEXTLINE
#define DECLARE_SETTING_ENUM_WITH_RENAME(NEW_NAME, ENUM_TYPE) \
struct SettingField##NEW_NAME##Traits \
{ \
using EnumType = ENUM_TYPE; \
using EnumValuePairs = std::pair<const char *, EnumType>[]; \
static const String & toString(EnumType value); \
static EnumType fromString(std::string_view str); \
}; \
@ -391,13 +398,20 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
using SettingField##NEW_NAME = SettingFieldEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>;
/// NOLINTNEXTLINE
#define IMPLEMENT_SETTING_ENUM_WITH_RENAME(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \
#define IMPLEMENT_SETTING_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \
IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, EnumValuePairs, __VA_ARGS__)
/// NOLINTNEXTLINE
#define IMPLEMENT_SETTING_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME) \
IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, , getEnumValues<EnumType>())
/// NOLINTNEXTLINE
#define IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, PAIRS_TYPE, ...) \
const String & SettingField##NEW_NAME##Traits::toString(typename SettingField##NEW_NAME::EnumType value) \
{ \
static const std::unordered_map<EnumType, String> map = [] { \
std::unordered_map<EnumType, String> res; \
constexpr std::pair<const char *, EnumType> pairs[] = __VA_ARGS__; \
for (const auto & [name, val] : pairs) \
for (const auto & [name, val] : PAIRS_TYPE __VA_ARGS__) \
res.emplace(val, name); \
return res; \
}(); \
@ -413,8 +427,7 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
{ \
static const std::unordered_map<std::string_view, EnumType> map = [] { \
std::unordered_map<std::string_view, EnumType> res; \
constexpr std::pair<const char *, EnumType> pairs[] = __VA_ARGS__; \
for (const auto & [name, val] : pairs) \
for (const auto & [name, val] : PAIRS_TYPE __VA_ARGS__) \
res.emplace(name, val); \
return res; \
}(); \
@ -527,6 +540,7 @@ void SettingFieldMultiEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
struct SettingField##NEW_NAME##Traits \
{ \
using EnumType = ENUM_TYPE; \
using EnumValuePairs = std::pair<const char *, EnumType>[]; \
static size_t getEnumSize(); \
static const String & toString(EnumType value); \
static EnumType fromString(std::string_view str); \
@ -540,11 +554,54 @@ void SettingFieldMultiEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
/// NOLINTNEXTLINE
#define IMPLEMENT_SETTING_MULTI_ENUM_WITH_RENAME(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \
IMPLEMENT_SETTING_ENUM_WITH_RENAME(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__)\
IMPLEMENT_SETTING_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__)\
size_t SettingField##NEW_NAME##Traits::getEnumSize() {\
return std::initializer_list<std::pair<const char*, NEW_NAME>> __VA_ARGS__ .size();\
}
/// NOLINTNEXTLINE
#define IMPLEMENT_SETTING_MULTI_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME) \
IMPLEMENT_SETTING_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME)\
size_t SettingField##NEW_NAME##Traits::getEnumSize() {\
return getEnumValues<EnumType>().size();\
}
/// Setting field for specifying user-defined timezone. It is basically a string, but it needs validation.
struct SettingFieldTimezone
{
String value;
bool changed = false;
explicit SettingFieldTimezone(std::string_view str = {}) { validateTimezone(std::string(str)); value = str; }
explicit SettingFieldTimezone(const String & str) { validateTimezone(str); value = str; }
explicit SettingFieldTimezone(String && str) { validateTimezone(str); value = std::move(str); }
explicit SettingFieldTimezone(const char * str) { validateTimezone(str); value = str; }
explicit SettingFieldTimezone(const Field & f) { const String & str = f.safeGet<const String &>(); validateTimezone(str); value = str; }
SettingFieldTimezone & operator =(std::string_view str) { validateTimezone(std::string(str)); value = str; changed = true; return *this; }
SettingFieldTimezone & operator =(const String & str) { *this = std::string_view{str}; return *this; }
SettingFieldTimezone & operator =(String && str) { validateTimezone(str); value = std::move(str); changed = true; return *this; }
SettingFieldTimezone & operator =(const char * str) { *this = std::string_view{str}; return *this; }
SettingFieldTimezone & operator =(const Field & f) { *this = f.safeGet<const String &>(); return *this; }
operator const String &() const { return value; } /// NOLINT
explicit operator Field() const { return value; }
const String & toString() const { return value; }
void parseFromString(const String & str) { *this = str; }
void writeBinary(WriteBuffer & out) const;
void readBinary(ReadBuffer & in);
private:
void validateTimezone(const std::string & tz_str)
{
cctz::time_zone validated_tz;
if (!tz_str.empty() && !cctz::load_time_zone(tz_str, &validated_tz))
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid time zone: {}", tz_str);
}
};
/// Can keep a value of any type. Used for user-defined settings.
struct SettingFieldCustom
{

View File

@ -1046,7 +1046,7 @@ void BaseDaemon::shouldSetupWatchdog(char * argv0_)
void BaseDaemon::setupWatchdog()
{
/// Initialize in advance to avoid double initialization in forked processes.
DateLUT::instance();
DateLUT::serverTimezoneInstance();
std::string original_process_name;
if (argv0)

View File

@ -21,7 +21,9 @@ namespace DB
* all types with different time zones are equivalent and may be used interchangingly.
* Time zone only affects parsing and displaying in text formats.
*
* If time zone is not specified (example: DateTime without parameter), then default time zone is used.
* If time zone is not specified (example: DateTime without parameter),
* then `session_timezone` setting value is used.
* If `session_timezone` is not set (or empty string), server default time zone is used.
* Default time zone is server time zone, if server is doing transformations
* and if client is doing transformations, unless 'use_client_time_zone' setting is passed to client;
* Server time zone is the time zone specified in 'timezone' parameter in configuration file,

View File

@ -123,9 +123,9 @@ llvm::Value * nativeCast(llvm::IRBuilderBase & b, const DataTypePtr & from_type,
}
else if (to_type->isNullable())
{
auto * from_native_type = toNativeType(b, from_type);
auto * to_native_type = toNativeType(b, to_type);
auto * inner = nativeCast(b, from_type, value, removeNullable(to_type));
return b.CreateInsertValue(llvm::Constant::getNullValue(from_native_type), inner, {0});
return b.CreateInsertValue(llvm::Constant::getNullValue(to_native_type), inner, {0});
}
else
{

View File

@ -13,7 +13,7 @@ namespace DB
void SerializationDate::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
{
writeDateText(DayNum(assert_cast<const ColumnUInt16 &>(column).getData()[row_num]), ostr);
writeDateText(DayNum(assert_cast<const ColumnUInt16 &>(column).getData()[row_num]), ostr, time_zone);
}
void SerializationDate::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
@ -26,7 +26,7 @@ void SerializationDate::deserializeWholeText(IColumn & column, ReadBuffer & istr
void SerializationDate::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
{
DayNum x;
readDateText(x, istr);
readDateText(x, istr, time_zone);
assert_cast<ColumnUInt16 &>(column).getData().push_back(x);
}
@ -46,7 +46,7 @@ void SerializationDate::deserializeTextQuoted(IColumn & column, ReadBuffer & ist
{
DayNum x;
assertChar('\'', istr);
readDateText(x, istr);
readDateText(x, istr, time_zone);
assertChar('\'', istr);
assert_cast<ColumnUInt16 &>(column).getData().push_back(x); /// It's important to do this at the end - for exception safety.
}
@ -62,7 +62,7 @@ void SerializationDate::deserializeTextJSON(IColumn & column, ReadBuffer & istr,
{
DayNum x;
assertChar('"', istr);
readDateText(x, istr);
readDateText(x, istr, time_zone);
assertChar('"', istr);
assert_cast<ColumnUInt16 &>(column).getData().push_back(x);
}
@ -77,8 +77,12 @@ void SerializationDate::serializeTextCSV(const IColumn & column, size_t row_num,
void SerializationDate::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
{
DayNum value;
readCSV(value, istr);
readCSV(value, istr, time_zone);
assert_cast<ColumnUInt16 &>(column).getData().push_back(value);
}
SerializationDate::SerializationDate(const DateLUTImpl & time_zone_) : time_zone(time_zone_)
{
}
}

View File

@ -1,6 +1,7 @@
#pragma once
#include <DataTypes/Serializations/SerializationNumber.h>
#include <Common/DateLUT.h>
namespace DB
{
@ -8,6 +9,8 @@ namespace DB
class SerializationDate final : public SerializationNumber<UInt16>
{
public:
explicit SerializationDate(const DateLUTImpl & time_zone_ = DateLUT::instance());
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
@ -18,6 +21,9 @@ public:
void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
protected:
const DateLUTImpl & time_zone;
};
}

View File

@ -11,7 +11,7 @@ namespace DB
void SerializationDate32::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
{
writeDateText(ExtendedDayNum(assert_cast<const ColumnInt32 &>(column).getData()[row_num]), ostr);
writeDateText(ExtendedDayNum(assert_cast<const ColumnInt32 &>(column).getData()[row_num]), ostr, time_zone);
}
void SerializationDate32::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
@ -24,7 +24,7 @@ void SerializationDate32::deserializeWholeText(IColumn & column, ReadBuffer & is
void SerializationDate32::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
{
ExtendedDayNum x;
readDateText(x, istr);
readDateText(x, istr, time_zone);
assert_cast<ColumnInt32 &>(column).getData().push_back(x);
}
@ -44,7 +44,7 @@ void SerializationDate32::deserializeTextQuoted(IColumn & column, ReadBuffer & i
{
ExtendedDayNum x;
assertChar('\'', istr);
readDateText(x, istr);
readDateText(x, istr, time_zone);
assertChar('\'', istr);
assert_cast<ColumnInt32 &>(column).getData().push_back(x); /// It's important to do this at the end - for exception safety.
}
@ -60,7 +60,7 @@ void SerializationDate32::deserializeTextJSON(IColumn & column, ReadBuffer & ist
{
ExtendedDayNum x;
assertChar('"', istr);
readDateText(x, istr);
readDateText(x, istr, time_zone);
assertChar('"', istr);
assert_cast<ColumnInt32 &>(column).getData().push_back(x);
}
@ -78,4 +78,8 @@ void SerializationDate32::deserializeTextCSV(IColumn & column, ReadBuffer & istr
readCSV(value, istr);
assert_cast<ColumnInt32 &>(column).getData().push_back(value.getExtenedDayNum());
}
SerializationDate32::SerializationDate32(const DateLUTImpl & time_zone_) : time_zone(time_zone_)
{
}
}

View File

@ -1,12 +1,15 @@
#pragma once
#include <DataTypes/Serializations/SerializationNumber.h>
#include <Common/DateLUT.h>
namespace DB
{
class SerializationDate32 final : public SerializationNumber<Int32>
{
public:
explicit SerializationDate32(const DateLUTImpl & time_zone_ = DateLUT::instance());
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
@ -17,5 +20,8 @@ public:
void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
protected:
const DateLUTImpl & time_zone;
};
}

View File

@ -710,8 +710,9 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, Contex
static UUID getTableUUIDIfReplicated(const String & metadata, ContextPtr context)
{
bool looks_like_replicated = metadata.find("ReplicatedMergeTree") != std::string::npos;
if (!looks_like_replicated)
bool looks_like_replicated = metadata.find("Replicated") != std::string::npos;
bool looks_like_merge_tree = metadata.find("MergeTree") != std::string::npos;
if (!looks_like_replicated || !looks_like_merge_tree)
return UUIDHelpers::Nil;
ParserCreateQuery parser;
@ -984,7 +985,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep
const auto & create_query_string = metadata_it->second;
if (isTableExist(table_name, getContext()))
{
assert(create_query_string == readMetadataFile(table_name));
assert(create_query_string == readMetadataFile(table_name) || getTableUUIDIfReplicated(create_query_string, getContext()) != UUIDHelpers::Nil);
continue;
}
@ -1180,7 +1181,7 @@ void DatabaseReplicated::dropTable(ContextPtr local_context, const String & tabl
std::lock_guard lock{metadata_mutex};
UInt64 new_digest = tables_metadata_digest;
new_digest -= getMetadataHash(table_name);
if (txn && !txn->isCreateOrReplaceQuery())
if (txn && !txn->isCreateOrReplaceQuery() && !is_recovering)
txn->addOp(zkutil::makeSetRequest(replica_path + "/digest", toString(new_digest), -1));
DatabaseAtomic::dropTableImpl(local_context, table_name, sync);
@ -1234,7 +1235,7 @@ void DatabaseReplicated::renameTable(ContextPtr local_context, const String & ta
new_digest -= DB::getMetadataHash(to_table_name, statement_to);
new_digest += DB::getMetadataHash(table_name, statement_to);
}
if (txn)
if (txn && !is_recovering)
txn->addOp(zkutil::makeSetRequest(replica_path + "/digest", toString(new_digest), -1));
DatabaseAtomic::renameTable(local_context, table_name, to_database, to_table_name, exchange, dictionary);
@ -1260,7 +1261,7 @@ void DatabaseReplicated::commitCreateTable(const ASTCreateQuery & query, const S
std::lock_guard lock{metadata_mutex};
UInt64 new_digest = tables_metadata_digest;
new_digest += DB::getMetadataHash(query.getTable(), statement);
if (txn && !txn->isCreateOrReplaceQuery())
if (txn && !txn->isCreateOrReplaceQuery() && !is_recovering)
txn->addOp(zkutil::makeSetRequest(replica_path + "/digest", toString(new_digest), -1));
DatabaseAtomic::commitCreateTable(query, table, table_metadata_tmp_path, table_metadata_path, query_context);
@ -1273,7 +1274,7 @@ void DatabaseReplicated::commitAlterTable(const StorageID & table_id,
const String & statement, ContextPtr query_context)
{
auto txn = query_context->getZooKeeperMetadataTransaction();
assert(!ddl_worker->isCurrentlyActive() || txn);
assert(!ddl_worker || !ddl_worker->isCurrentlyActive() || txn);
if (txn && txn->isInitialQuery())
{
String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(table_id.table_name);
@ -1284,7 +1285,7 @@ void DatabaseReplicated::commitAlterTable(const StorageID & table_id,
UInt64 new_digest = tables_metadata_digest;
new_digest -= getMetadataHash(table_id.table_name);
new_digest += DB::getMetadataHash(table_id.table_name, statement);
if (txn)
if (txn && !is_recovering)
txn->addOp(zkutil::makeSetRequest(replica_path + "/digest", toString(new_digest), -1));
DatabaseAtomic::commitAlterTable(table_id, table_metadata_tmp_path, table_metadata_path, statement, query_context);
@ -1307,7 +1308,7 @@ void DatabaseReplicated::detachTablePermanently(ContextPtr local_context, const
std::lock_guard lock{metadata_mutex};
UInt64 new_digest = tables_metadata_digest;
new_digest -= getMetadataHash(table_name);
if (txn)
if (txn && !is_recovering)
txn->addOp(zkutil::makeSetRequest(replica_path + "/digest", toString(new_digest), -1));
DatabaseAtomic::detachTablePermanently(local_context, table_name);
@ -1331,7 +1332,7 @@ void DatabaseReplicated::removeDetachedPermanentlyFlag(ContextPtr local_context,
if (attach)
{
new_digest += getMetadataHash(table_name);
if (txn)
if (txn && !is_recovering)
txn->addOp(zkutil::makeSetRequest(replica_path + "/digest", toString(new_digest), -1));
}

View File

@ -91,6 +91,7 @@ void DatabaseReplicatedDDLWorker::initializeReplication()
if (zookeeper->tryGet(database->replica_path + "/digest", digest_str))
{
digest = parse<UInt64>(digest_str);
LOG_TRACE(log, "Metadata digest in ZooKeeper: {}", digest);
std::lock_guard lock{database->metadata_mutex};
local_digest = database->tables_metadata_digest;
}

View File

@ -114,7 +114,10 @@ QueryPipeline ExecutableDictionarySource::loadAll()
auto command = configuration.command;
updateCommandIfNeeded(command, coordinator_configuration.execute_direct, context);
return QueryPipeline(coordinator->createPipe(command, configuration.command_arguments, sample_block, context));
ShellCommandSourceConfiguration command_configuration {
.check_exit_code = true,
};
return QueryPipeline(coordinator->createPipe(command, configuration.command_arguments, {}, sample_block, context, command_configuration));
}
QueryPipeline ExecutableDictionarySource::loadUpdatedAll()
@ -148,7 +151,11 @@ QueryPipeline ExecutableDictionarySource::loadUpdatedAll()
update_time = new_update_time;
LOG_TRACE(log, "loadUpdatedAll {}", command);
return QueryPipeline(coordinator->createPipe(command, command_arguments, sample_block, context));
ShellCommandSourceConfiguration command_configuration {
.check_exit_code = true,
};
return QueryPipeline(coordinator->createPipe(command, command_arguments, {}, sample_block, context, command_configuration));
}
QueryPipeline ExecutableDictionarySource::loadIds(const std::vector<UInt64> & ids)
@ -179,7 +186,11 @@ QueryPipeline ExecutableDictionarySource::getStreamForBlock(const Block & block)
Pipes shell_input_pipes;
shell_input_pipes.emplace_back(std::move(shell_input_pipe));
auto pipe = coordinator->createPipe(command, configuration.command_arguments, std::move(shell_input_pipes), sample_block, context);
ShellCommandSourceConfiguration command_configuration {
.check_exit_code = true,
};
auto pipe = coordinator->createPipe(command, configuration.command_arguments, std::move(shell_input_pipes), sample_block, context, command_configuration);
if (configuration.implicit_key)
pipe.addTransform(std::make_shared<TransformWithAdditionalColumns>(block, pipe.getHeader()));

View File

@ -132,6 +132,7 @@ QueryPipeline ExecutablePoolDictionarySource::getStreamForBlock(const Block & bl
ShellCommandSourceConfiguration command_configuration;
command_configuration.read_fixed_number_of_rows = true;
command_configuration.number_of_rows_to_read = block.rows();
command_configuration.check_exit_code = true;
Pipes shell_input_pipes;
shell_input_pipes.emplace_back(std::move(shell_input_pipe));

View File

@ -153,6 +153,7 @@ public:
/// * Parallel reading.
/// To enable it, make sure `buf` is a SeekableReadBuffer implementing readBigAt().
/// * Parallel parsing.
/// `buf` must outlive the returned IInputFormat.
InputFormatPtr getInput(
const String & name,
ReadBuffer & buf,

View File

@ -434,7 +434,7 @@ struct ToDate32Transform32Or64Signed
static NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone)
{
static const Int32 daynum_min_offset = -static_cast<Int32>(DateLUT::instance().getDayNumOffsetEpoch());
static const Int32 daynum_min_offset = -static_cast<Int32>(time_zone.getDayNumOffsetEpoch());
if (from < daynum_min_offset)
return daynum_min_offset;
return (from < DATE_LUT_MAX_EXTEND_DAY_NUM)
@ -713,9 +713,9 @@ template <>
struct FormatImpl<DataTypeDate>
{
template <typename ReturnType = void>
static ReturnType execute(const DataTypeDate::FieldType x, WriteBuffer & wb, const DataTypeDate *, const DateLUTImpl *)
static ReturnType execute(const DataTypeDate::FieldType x, WriteBuffer & wb, const DataTypeDate *, const DateLUTImpl * time_zone)
{
writeDateText(DayNum(x), wb);
writeDateText(DayNum(x), wb, *time_zone);
return ReturnType(true);
}
};
@ -724,9 +724,9 @@ template <>
struct FormatImpl<DataTypeDate32>
{
template <typename ReturnType = void>
static ReturnType execute(const DataTypeDate32::FieldType x, WriteBuffer & wb, const DataTypeDate32 *, const DateLUTImpl *)
static ReturnType execute(const DataTypeDate32::FieldType x, WriteBuffer & wb, const DataTypeDate32 *, const DateLUTImpl * time_zone)
{
writeDateText(ExtendedDayNum(x), wb);
writeDateText(ExtendedDayNum(x), wb, *time_zone);
return ReturnType(true);
}
};
@ -825,7 +825,10 @@ struct ConvertImpl<FromDataType, DataTypeString, Name, ConvertDefaultBehaviorTag
const auto & type = static_cast<const FromDataType &>(*col_with_type_and_name.type);
const DateLUTImpl * time_zone = nullptr;
/// For argument of DateTime type, second argument with time zone could be specified.
if constexpr (std::is_same_v<FromDataType, DataTypeDate> || std::is_same_v<FromDataType, DataTypeDate32>)
time_zone = &DateLUT::instance();
/// For argument of Date or DateTime type, second argument with time zone could be specified.
if constexpr (std::is_same_v<FromDataType, DataTypeDateTime> || std::is_same_v<FromDataType, DataTypeDateTime64>)
{
auto non_null_args = createBlockWithNestedColumns(arguments);
@ -976,18 +979,18 @@ void parseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateLUTI
}
template <>
inline void parseImpl<DataTypeDate>(DataTypeDate::FieldType & x, ReadBuffer & rb, const DateLUTImpl *)
inline void parseImpl<DataTypeDate>(DataTypeDate::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone)
{
DayNum tmp(0);
readDateText(tmp, rb);
readDateText(tmp, rb, *time_zone);
x = tmp;
}
template <>
inline void parseImpl<DataTypeDate32>(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl *)
inline void parseImpl<DataTypeDate32>(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone)
{
ExtendedDayNum tmp(0);
readDateText(tmp, rb);
readDateText(tmp, rb, *time_zone);
x = tmp;
}
@ -1035,20 +1038,20 @@ bool tryParseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateL
}
template <>
inline bool tryParseImpl<DataTypeDate>(DataTypeDate::FieldType & x, ReadBuffer & rb, const DateLUTImpl *)
inline bool tryParseImpl<DataTypeDate>(DataTypeDate::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone)
{
DayNum tmp(0);
if (!tryReadDateText(tmp, rb))
if (!tryReadDateText(tmp, rb, *time_zone))
return false;
x = tmp;
return true;
}
template <>
inline bool tryParseImpl<DataTypeDate32>(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl *)
inline bool tryParseImpl<DataTypeDate32>(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone)
{
ExtendedDayNum tmp(0);
if (!tryReadDateText(tmp, rb))
if (!tryReadDateText(tmp, rb, *time_zone))
return false;
x = tmp;
return true;
@ -1188,7 +1191,7 @@ struct ConvertThroughParsing
const DateLUTImpl * local_time_zone [[maybe_unused]] = nullptr;
const DateLUTImpl * utc_time_zone [[maybe_unused]] = nullptr;
/// For conversion to DateTime type, second argument with time zone could be specified.
/// For conversion to Date or DateTime type, second argument with time zone could be specified.
if constexpr (std::is_same_v<ToDataType, DataTypeDateTime> || to_datetime64)
{
const auto result_type = removeNullable(res_type);
@ -1201,6 +1204,12 @@ struct ConvertThroughParsing
if constexpr (parsing_mode == ConvertFromStringParsingMode::BestEffort || parsing_mode == ConvertFromStringParsingMode::BestEffortUS)
utc_time_zone = &DateLUT::instance("UTC");
}
else if constexpr (std::is_same_v<ToDataType, DataTypeDate> || std::is_same_v<ToDataType, DataTypeDate32>)
{
// Timezone is more or less dummy when parsing Date/Date32 from string.
local_time_zone = &DateLUT::instance();
utc_time_zone = &DateLUT::instance("UTC");
}
const IColumn * col_from = arguments[0].column.get();
const ColumnString * col_from_string = checkAndGetColumn<ColumnString>(col_from);
@ -1751,7 +1760,7 @@ public:
|| std::is_same_v<Name, NameToUnixTimestamp>
// toDate(value[, timezone : String])
|| std::is_same_v<ToDataType, DataTypeDate> // TODO: shall we allow timestamp argument for toDate? DateTime knows nothing about timezones and this argument is ignored below.
// toDate(value[, timezone : String])
// toDate32(value[, timezone : String])
|| std::is_same_v<ToDataType, DataTypeDate32>
// toDateTime(value[, timezone: String])
|| std::is_same_v<ToDataType, DataTypeDateTime>

View File

@ -60,13 +60,22 @@ namespace
};
/// Returns the server time zone.
/// Returns timezone for current session.
class FunctionTimezone : public FunctionConstantBase<FunctionTimezone, String, DataTypeString>
{
public:
static constexpr auto name = "timezone";
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionTimezone>(context); }
explicit FunctionTimezone(ContextPtr context) : FunctionConstantBase(String{DateLUT::instance().getTimeZone()}, context->isDistributed()) {}
explicit FunctionTimezone(ContextPtr context) : FunctionConstantBase(DateLUT::instance().getTimeZone(), context->isDistributed()) {}
};
/// Returns the server time zone (timezone in which server runs).
class FunctionServerTimezone : public FunctionConstantBase<FunctionServerTimezone, String, DataTypeString>
{
public:
static constexpr auto name = "serverTimezone";
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionServerTimezone>(context); }
explicit FunctionServerTimezone(ContextPtr context) : FunctionConstantBase(DateLUT::serverTimezoneInstance().getTimeZone(), context->isDistributed()) {}
};
@ -151,8 +160,34 @@ REGISTER_FUNCTION(TcpPort)
REGISTER_FUNCTION(Timezone)
{
factory.registerFunction<FunctionTimezone>();
factory.registerAlias("timeZone", "timezone");
factory.registerFunction<FunctionTimezone>(
FunctionDocumentation{
.description=R"(
Returns the default timezone for current session.
Used as default timezone for parsing DateTime|DateTime64 without explicitly specified timezone.
Can be changed with SET timezone = 'New/Tz'
[example:timezone]
)",
.examples{{"timezone", "SELECT timezone();", ""}},
.categories{"Constant", "Miscellaneous"}
});
factory.registerAlias("timeZone", "timezone");
}
REGISTER_FUNCTION(ServerTimezone)
{
factory.registerFunction<FunctionServerTimezone>(
FunctionDocumentation{
.description=R"(
Returns the timezone name in which server operates.
[example:serverTimezone]
)",
.examples{{"serverTimezone", "SELECT serverTimezone();", ""}},
.categories{"Constant", "Miscellaneous"}
});
factory.registerAlias("serverTimeZone", "serverTimezone");
}
REGISTER_FUNCTION(Uptime)

View File

@ -6,7 +6,6 @@
#include <Common/DateLUTImpl.h>
#include <Core/Field.h>
namespace DB
{

View File

@ -256,7 +256,7 @@ void ParallelReadBuffer::readerThreadFunction(ReadWorkerPtr read_worker)
return false;
};
size_t r = input.readBigAt(read_worker->segment.data(), read_worker->segment.size(), read_worker->start_offset);
size_t r = input.readBigAt(read_worker->segment.data(), read_worker->segment.size(), read_worker->start_offset, on_progress);
if (!on_progress(r) && r < read_worker->segment.size())
throw Exception(

View File

@ -702,7 +702,7 @@ inline void convertToDayNum(DayNum & date, ExtendedDayNum & from)
}
template <typename ReturnType = void>
inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf)
inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut)
{
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
@ -713,13 +713,13 @@ inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf)
else if (!readDateTextImpl<ReturnType>(local_date, buf))
return false;
ExtendedDayNum ret = DateLUT::instance().makeDayNum(local_date.year(), local_date.month(), local_date.day());
convertToDayNum(date,ret);
ExtendedDayNum ret = date_lut.makeDayNum(local_date.year(), local_date.month(), local_date.day());
convertToDayNum(date, ret);
return ReturnType(true);
}
template <typename ReturnType = void>
inline ReturnType readDateTextImpl(ExtendedDayNum & date, ReadBuffer & buf)
inline ReturnType readDateTextImpl(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut)
{
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
@ -731,7 +731,7 @@ inline ReturnType readDateTextImpl(ExtendedDayNum & date, ReadBuffer & buf)
return false;
/// When the parameter is out of rule or out of range, Date32 uses 1925-01-01 as the default value (-DateLUT::instance().getDayNumOffsetEpoch(), -16436) and Date uses 1970-01-01.
date = DateLUT::instance().makeDayNum(local_date.year(), local_date.month(), local_date.day(), -static_cast<Int32>(DateLUT::instance().getDayNumOffsetEpoch()));
date = date_lut.makeDayNum(local_date.year(), local_date.month(), local_date.day(), -static_cast<Int32>(date_lut.getDayNumOffsetEpoch()));
return ReturnType(true);
}
@ -741,14 +741,14 @@ inline void readDateText(LocalDate & date, ReadBuffer & buf)
readDateTextImpl<void>(date, buf);
}
inline void readDateText(DayNum & date, ReadBuffer & buf)
inline void readDateText(DayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance())
{
readDateTextImpl<void>(date, buf);
readDateTextImpl<void>(date, buf, date_lut);
}
inline void readDateText(ExtendedDayNum & date, ReadBuffer & buf)
inline void readDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance())
{
readDateTextImpl<void>(date, buf);
readDateTextImpl<void>(date, buf, date_lut);
}
inline bool tryReadDateText(LocalDate & date, ReadBuffer & buf)
@ -756,14 +756,14 @@ inline bool tryReadDateText(LocalDate & date, ReadBuffer & buf)
return readDateTextImpl<bool>(date, buf);
}
inline bool tryReadDateText(DayNum & date, ReadBuffer & buf)
inline bool tryReadDateText(DayNum & date, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance())
{
return readDateTextImpl<bool>(date, buf);
return readDateTextImpl<bool>(date, buf, time_zone);
}
inline bool tryReadDateText(ExtendedDayNum & date, ReadBuffer & buf)
inline bool tryReadDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance())
{
return readDateTextImpl<bool>(date, buf);
return readDateTextImpl<bool>(date, buf, time_zone);
}
UUID parseUUID(std::span<const UInt8> src);
@ -1135,8 +1135,10 @@ inline bool tryReadText(IPv6 & x, ReadBuffer & buf) { return tryReadIPv6Text(x,
inline void readText(is_floating_point auto & x, ReadBuffer & buf) { readFloatText(x, buf); }
inline void readText(String & x, ReadBuffer & buf) { readEscapedString(x, buf); }
inline void readText(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { readDateText(x, buf, time_zone); }
inline void readText(LocalDate & x, ReadBuffer & buf) { readDateText(x, buf); }
inline void readText(DayNum & x, ReadBuffer & buf) { readDateText(x, buf); }
inline void readText(LocalDateTime & x, ReadBuffer & buf) { readDateTimeText(x, buf); }
inline void readText(UUID & x, ReadBuffer & buf) { readUUIDText(x, buf); }
inline void readText(IPv4 & x, ReadBuffer & buf) { readIPv4Text(x, buf); }
@ -1148,6 +1150,10 @@ template <typename T>
requires is_arithmetic_v<T>
inline void readQuoted(T & x, ReadBuffer & buf) { readText(x, buf); }
template <typename T>
requires is_arithmetic_v<T>
inline void readQuoted(T & x, ReadBuffer & buf, const DateLUTImpl & time_zone) { readText(x, buf, time_zone); }
inline void readQuoted(String & x, ReadBuffer & buf) { readQuotedString(x, buf); }
inline void readQuoted(LocalDate & x, ReadBuffer & buf)
@ -1190,6 +1196,10 @@ template <typename T>
requires is_arithmetic_v<T>
inline void readDoubleQuoted(T & x, ReadBuffer & buf) { readText(x, buf); }
template <typename T>
requires is_arithmetic_v<T>
inline void readDoubleQuoted(T & x, ReadBuffer & buf, const DateLUTImpl & time_zone) { readText(x, buf, time_zone); }
inline void readDoubleQuoted(String & x, ReadBuffer & buf) { readDoubleQuotedString(x, buf); }
inline void readDoubleQuoted(LocalDate & x, ReadBuffer & buf)
@ -1206,7 +1216,7 @@ inline void readDoubleQuoted(LocalDateTime & x, ReadBuffer & buf)
assertChar('"', buf);
}
/// CSV, for numbers, dates: quotes are optional, no special escaping rules.
/// CSV for numbers: quotes are optional, no special escaping rules.
template <typename T>
inline void readCSVSimple(T & x, ReadBuffer & buf)
{
@ -1224,6 +1234,24 @@ inline void readCSVSimple(T & x, ReadBuffer & buf)
assertChar(maybe_quote, buf);
}
// standalone overload for dates: to avoid instantiating DateLUTs while parsing other types
template <typename T>
inline void readCSVSimple(T & x, ReadBuffer & buf, const DateLUTImpl & time_zone)
{
if (buf.eof()) [[unlikely]]
throwReadAfterEOF();
char maybe_quote = *buf.position();
if (maybe_quote == '\'' || maybe_quote == '\"')
++buf.position();
readText(x, buf, time_zone);
if (maybe_quote == '\'' || maybe_quote == '\"')
assertChar(maybe_quote, buf);
}
template <typename T>
requires is_arithmetic_v<T>
inline void readCSV(T & x, ReadBuffer & buf)
@ -1234,6 +1262,7 @@ inline void readCSV(T & x, ReadBuffer & buf)
inline void readCSV(String & x, ReadBuffer & buf, const FormatSettings::CSV & settings) { readCSVString(x, buf, settings); }
inline void readCSV(LocalDate & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
inline void readCSV(DayNum & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
inline void readCSV(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone) { readCSVSimple(x, buf, time_zone); }
inline void readCSV(LocalDateTime & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
inline void readCSV(UUID & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
inline void readCSV(IPv4 & x, ReadBuffer & buf) { readCSVSimple(x, buf); }

View File

@ -1,4 +1,5 @@
#include <IO/WriteBufferFromOStream.h>
#include <Common/logger_useful.h>
namespace DB
@ -18,7 +19,14 @@ void WriteBufferFromOStream::nextImpl()
ostr->flush();
if (!ostr->good())
throw Exception(ErrorCodes::CANNOT_WRITE_TO_OSTREAM, "Cannot write to ostream at offset {}", count());
{
/// FIXME do not call finalize in dtors (and remove iostreams)
bool avoid_throwing_exceptions = std::uncaught_exceptions();
if (avoid_throwing_exceptions)
LOG_ERROR(&Poco::Logger::get("WriteBufferFromOStream"), "Cannot write to ostream at offset {}. Stack trace: {}", count(), StackTrace().toString());
else
throw Exception(ErrorCodes::CANNOT_WRITE_TO_OSTREAM, "Cannot write to ostream at offset {}", count());
}
}
WriteBufferFromOStream::WriteBufferFromOStream(

View File

@ -697,15 +697,15 @@ inline void writeDateText(const LocalDate & date, WriteBuffer & buf)
}
template <char delimiter = '-'>
inline void writeDateText(DayNum date, WriteBuffer & buf)
inline void writeDateText(DayNum date, WriteBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance())
{
writeDateText<delimiter>(LocalDate(date), buf);
writeDateText<delimiter>(LocalDate(date, time_zone), buf);
}
template <char delimiter = '-'>
inline void writeDateText(ExtendedDayNum date, WriteBuffer & buf)
inline void writeDateText(ExtendedDayNum date, WriteBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance())
{
writeDateText<delimiter>(LocalDate(date), buf);
writeDateText<delimiter>(LocalDate(date, time_zone), buf);
}
/// In the format YYYY-MM-DD HH:MM:SS
@ -886,7 +886,7 @@ inline void writeText(is_enum auto x, WriteBuffer & buf) { writeText(magic_enum:
inline void writeText(std::string_view x, WriteBuffer & buf) { writeString(x.data(), x.size(), buf); }
inline void writeText(const DayNum & x, WriteBuffer & buf) { writeDateText(LocalDate(x), buf); }
inline void writeText(const DayNum & x, WriteBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { writeDateText(LocalDate(x, time_zone), buf); }
inline void writeText(const LocalDate & x, WriteBuffer & buf) { writeDateText(x, buf); }
inline void writeText(const LocalDateTime & x, WriteBuffer & buf) { writeDateTimeText(x, buf); }
inline void writeText(const UUID & x, WriteBuffer & buf) { writeUUIDText(x, buf); }

View File

@ -1876,10 +1876,10 @@ struct ConjunctionNodes
ActionsDAG::NodeRawConstPtrs rejected;
};
/// Take a node which result is predicate.
/// Take a node which result is a predicate.
/// Assuming predicate is a conjunction (probably, trivial).
/// Find separate conjunctions nodes. Split nodes into allowed and rejected sets.
/// Allowed predicate is a predicate which can be calculated using only nodes from allowed_nodes set.
/// Allowed predicate is a predicate which can be calculated using only nodes from the allowed_nodes set.
ConjunctionNodes getConjunctionNodes(ActionsDAG::Node * predicate, std::unordered_set<const ActionsDAG::Node *> allowed_nodes)
{
ConjunctionNodes conjunction;
@ -2113,9 +2113,9 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown(
Node * predicate = const_cast<Node *>(tryFindInOutputs(filter_name));
if (!predicate)
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Output nodes for ActionsDAG do not contain filter column name {}. DAG:\n{}",
filter_name,
dumpDAG());
"Output nodes for ActionsDAG do not contain filter column name {}. DAG:\n{}",
filter_name,
dumpDAG());
/// If condition is constant let's do nothing.
/// It means there is nothing to push down or optimization was already applied.
@ -2142,18 +2142,29 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown(
}
auto conjunction = getConjunctionNodes(predicate, allowed_nodes);
if (conjunction.rejected.size() == 1 && !conjunction.rejected.front()->result_type->equals(*predicate->result_type)
&& conjunction.allowed.front()->type == ActionType::COLUMN)
{
// No further optimization can be done
if (conjunction.allowed.empty())
return nullptr;
chassert(predicate->result_type);
if (conjunction.rejected.size() == 1)
{
chassert(conjunction.rejected.front()->result_type);
if (conjunction.allowed.front()->type == ActionType::COLUMN
&& !conjunction.rejected.front()->result_type->equals(*predicate->result_type))
{
/// No further optimization can be done
return nullptr;
}
}
auto actions = cloneActionsForConjunction(conjunction.allowed, all_inputs);
if (!actions)
return nullptr;
/// Now, when actions are created, update current DAG.
/// Now, when actions are created, update the current DAG.
if (conjunction.rejected.empty())
{

View File

@ -615,9 +615,9 @@ bool FileCache::tryReserve(FileSegment & file_segment, const size_t size)
struct EvictionCandidates
{
explicit EvictionCandidates(KeyMetadataPtr key_metadata_) : key_metadata(key_metadata_) {}
explicit EvictionCandidates(KeyMetadataPtr key_metadata_) : key_metadata(std::move(key_metadata_)) {}
void add(FileSegmentMetadataPtr candidate)
void add(const FileSegmentMetadataPtr & candidate)
{
candidate->removal_candidate = true;
candidates.push_back(candidate);
@ -638,7 +638,7 @@ bool FileCache::tryReserve(FileSegment & file_segment, const size_t size)
std::unordered_map<Key, EvictionCandidates> to_delete;
size_t freeable_space = 0, freeable_count = 0;
auto iterate_func = [&](LockedKey & locked_key, FileSegmentMetadataPtr segment_metadata)
auto iterate_func = [&](LockedKey & locked_key, const FileSegmentMetadataPtr & segment_metadata)
{
chassert(segment_metadata->file_segment->assertCorrectness());
@ -658,7 +658,7 @@ bool FileCache::tryReserve(FileSegment & file_segment, const size_t size)
it->second.add(segment_metadata);
freeable_space += segment_metadata->size();
freeable_count += 1;
++freeable_count;
return PriorityIterationResult::CONTINUE;
}
@ -683,7 +683,7 @@ bool FileCache::tryReserve(FileSegment & file_segment, const size_t size)
if (is_query_priority_overflow())
{
query_priority->iterate(
[&](LockedKey & locked_key, FileSegmentMetadataPtr segment_metadata)
[&](LockedKey & locked_key, const FileSegmentMetadataPtr & segment_metadata)
{ return is_query_priority_overflow() ? iterate_func(locked_key, segment_metadata) : PriorityIterationResult::BREAK; },
cache_lock);
@ -696,20 +696,28 @@ bool FileCache::tryReserve(FileSegment & file_segment, const size_t size)
file_segment.key(), file_segment.offset());
}
auto is_main_priority_overflow = [&]
auto is_main_priority_overflow = [main_priority_size_limit = main_priority->getSizeLimit(),
main_priority_elements_limit = main_priority->getElementsLimit(),
size,
&freeable_space,
&freeable_count,
&file_segment,
&cache_lock,
my_main_priority = this->main_priority.get(),
my_log = this->log]
{
/// max_size == 0 means unlimited cache size,
/// max_element_size == 0 means unlimited number of cache elements.
const bool is_overflow = (main_priority->getSizeLimit() != 0
&& (main_priority->getSize(cache_lock) + size - freeable_space > main_priority->getSizeLimit()))
|| (main_priority->getElementsLimit() != 0
&& freeable_count == 0 && main_priority->getElementsCount(cache_lock) == main_priority->getElementsLimit());
const bool is_overflow =
/// size_limit == 0 means unlimited cache size
(main_priority_size_limit != 0 && (my_main_priority->getSize(cache_lock) + size - freeable_space > main_priority_size_limit))
/// elements_limit == 0 means unlimited number of cache elements
|| (main_priority_elements_limit != 0 && freeable_count == 0
&& my_main_priority->getElementsCount(cache_lock) == main_priority_elements_limit);
LOG_TEST(
log, "Overflow: {}, size: {}, ready to remove: {} ({} in number), current cache size: {}/{}, elements: {}/{}, while reserving for {}:{}",
my_log, "Overflow: {}, size: {}, ready to remove: {} ({} in number), current cache size: {}/{}, elements: {}/{}, while reserving for {}:{}",
is_overflow, size, freeable_space, freeable_count,
main_priority->getSize(cache_lock), main_priority->getSizeLimit(),
main_priority->getElementsCount(cache_lock), main_priority->getElementsLimit(),
my_main_priority->getSize(cache_lock), my_main_priority->getSizeLimit(),
my_main_priority->getElementsCount(cache_lock), my_main_priority->getElementsLimit(),
file_segment.key(), file_segment.offset());
return is_overflow;
@ -718,7 +726,7 @@ bool FileCache::tryReserve(FileSegment & file_segment, const size_t size)
if (is_main_priority_overflow())
{
main_priority->iterate(
[&](LockedKey & locked_key, FileSegmentMetadataPtr segment_metadata)
[&](LockedKey & locked_key, const FileSegmentMetadataPtr & segment_metadata)
{ return is_main_priority_overflow() ? iterate_func(locked_key, segment_metadata) : PriorityIterationResult::BREAK; },
cache_lock);
@ -831,7 +839,7 @@ void FileCache::removeAllReleasable()
auto lock = lockCache();
main_priority->iterate([&](LockedKey & locked_key, FileSegmentMetadataPtr segment_metadata)
main_priority->iterate([&](LockedKey & locked_key, const FileSegmentMetadataPtr & segment_metadata)
{
if (segment_metadata->releasable())
{
@ -1065,7 +1073,7 @@ FileSegmentsHolderPtr FileCache::dumpQueue()
assertInitialized();
FileSegments file_segments;
main_priority->iterate([&](LockedKey &, FileSegmentMetadataPtr segment_metadata)
main_priority->iterate([&](LockedKey &, const FileSegmentMetadataPtr & segment_metadata)
{
file_segments.push_back(FileSegment::getSnapshot(segment_metadata->file_segment));
return PriorityIterationResult::CONTINUE;
@ -1105,7 +1113,7 @@ size_t FileCache::getFileSegmentsNum() const
void FileCache::assertCacheCorrectness()
{
auto lock = lockCache();
main_priority->iterate([&](LockedKey &, FileSegmentMetadataPtr segment_metadata)
main_priority->iterate([&](LockedKey &, const FileSegmentMetadataPtr & segment_metadata)
{
const auto & file_segment = *segment_metadata->file_segment;
UNUSED(file_segment);

View File

@ -50,7 +50,7 @@ public:
virtual Entry & getEntry() = 0;
virtual void annul() = 0;
virtual void invalidate() = 0;
virtual void updateSize(int64_t size) = 0;
};
@ -64,7 +64,7 @@ public:
CONTINUE,
REMOVE_AND_CONTINUE,
};
using IterateFunc = std::function<IterationResult(LockedKey &, FileSegmentMetadataPtr)>;
using IterateFunc = std::function<IterationResult(LockedKey &, const FileSegmentMetadataPtr &)>;
IFileCachePriority(size_t max_size_, size_t max_elements_) : max_size(max_size_), max_elements(max_elements_) {}

View File

@ -25,6 +25,14 @@ IFileCachePriority::Iterator LRUFileCachePriority::add(
const CacheGuard::Lock &)
{
const auto & key = key_metadata->key;
if (size == 0)
{
throw Exception(
ErrorCodes::LOGICAL_ERROR,
"Adding zero size entries to LRU queue is not allowed "
"(key: {}, offset: {})", key, offset);
}
#ifndef NDEBUG
for (const auto & entry : queue)
{
@ -48,10 +56,9 @@ IFileCachePriority::Iterator LRUFileCachePriority::add(
}
auto iter = queue.insert(queue.end(), Entry(key, offset, size, key_metadata));
current_size += size;
CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size);
CurrentMetrics::add(CurrentMetrics::FilesystemCacheElements);
updateSize(size);
updateElementsCount(1);
LOG_TEST(
log, "Added entry into LRU queue, key: {}, offset: {}, size: {}",
@ -62,13 +69,11 @@ IFileCachePriority::Iterator LRUFileCachePriority::add(
void LRUFileCachePriority::removeAll(const CacheGuard::Lock &)
{
CurrentMetrics::sub(CurrentMetrics::FilesystemCacheSize, current_size);
CurrentMetrics::sub(CurrentMetrics::FilesystemCacheElements, queue.size());
LOG_TEST(log, "Removed all entries from LRU queue");
updateSize(-current_size);
updateElementsCount(-current_elements_num);
queue.clear();
current_size = 0;
}
void LRUFileCachePriority::pop(const CacheGuard::Lock &)
@ -78,10 +83,12 @@ void LRUFileCachePriority::pop(const CacheGuard::Lock &)
LRUFileCachePriority::LRUQueueIterator LRUFileCachePriority::remove(LRUQueueIterator it)
{
current_size -= it->size;
CurrentMetrics::sub(CurrentMetrics::FilesystemCacheSize, it->size);
CurrentMetrics::sub(CurrentMetrics::FilesystemCacheElements);
/// If size is 0, entry is invalidated, current_elements_num was already updated.
if (it->size)
{
updateSize(-it->size);
updateElementsCount(-1);
}
LOG_TEST(
log, "Removed entry from LRU queue, key: {}, offset: {}, size: {}",
@ -90,6 +97,19 @@ LRUFileCachePriority::LRUQueueIterator LRUFileCachePriority::remove(LRUQueueIter
return queue.erase(it);
}
void LRUFileCachePriority::updateSize(int64_t size)
{
current_size += size;
CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size);
}
void LRUFileCachePriority::updateElementsCount(int64_t num)
{
current_elements_num += num;
CurrentMetrics::add(CurrentMetrics::FilesystemCacheElements, num);
}
LRUFileCachePriority::LRUFileCacheIterator::LRUFileCacheIterator(
LRUFileCachePriority * cache_priority_,
LRUFileCachePriority::LRUQueueIterator queue_iter_)
@ -153,10 +173,16 @@ LRUFileCachePriority::LRUFileCacheIterator::remove(const CacheGuard::Lock &)
cache_priority, cache_priority->remove(queue_iter));
}
void LRUFileCachePriority::LRUFileCacheIterator::annul()
void LRUFileCachePriority::LRUFileCacheIterator::invalidate()
{
updateSize(-queue_iter->size);
chassert(queue_iter->size == 0);
LOG_TEST(
cache_priority->log,
"Invalidating entry in LRU queue. Key: {}, offset: {}, previous size: {}",
queue_iter->key, queue_iter->offset, queue_iter->size);
cache_priority->updateSize(-queue_iter->size);
cache_priority->updateElementsCount(-1);
queue_iter->size = 0;
}
void LRUFileCachePriority::LRUFileCacheIterator::updateSize(int64_t size)
@ -166,13 +192,8 @@ void LRUFileCachePriority::LRUFileCacheIterator::updateSize(int64_t size)
"Update size with {} in LRU queue for key: {}, offset: {}, previous size: {}",
size, queue_iter->key, queue_iter->offset, queue_iter->size);
cache_priority->current_size += size;
cache_priority->updateSize(size);
queue_iter->size += size;
CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size);
chassert(cache_priority->current_size >= 0);
chassert(queue_iter->size >= 0);
}
size_t LRUFileCachePriority::LRUFileCacheIterator::use(const CacheGuard::Lock &)

View File

@ -22,7 +22,7 @@ public:
size_t getSize(const CacheGuard::Lock &) const override { return current_size; }
size_t getElementsCount(const CacheGuard::Lock &) const override { return queue.size(); }
size_t getElementsCount(const CacheGuard::Lock &) const override { return current_elements_num; }
Iterator add(KeyMetadataPtr key_metadata, size_t offset, size_t size, const CacheGuard::Lock &) override;
@ -33,10 +33,16 @@ public:
void iterate(IterateFunc && func, const CacheGuard::Lock &) override;
private:
void updateElementsCount(int64_t num);
void updateSize(int64_t size);
LRUQueue queue;
Poco::Logger * log = &Poco::Logger::get("LRUFileCachePriority");
std::atomic<size_t> current_size = 0;
/// current_elements_num is not always equal to queue.size()
/// because of invalidated entries.
std::atomic<size_t> current_elements_num = 0;
LRUQueueIterator remove(LRUQueueIterator it);
};
@ -56,7 +62,7 @@ public:
Iterator remove(const CacheGuard::Lock &) override;
void annul() override;
void invalidate() override;
void updateSize(int64_t size) override;

View File

@ -53,11 +53,13 @@ KeyMetadata::KeyMetadata(
const Key & key_,
const std::string & key_path_,
CleanupQueue & cleanup_queue_,
Poco::Logger * log_,
bool created_base_directory_)
: key(key_)
, key_path(key_path_)
, cleanup_queue(cleanup_queue_)
, created_base_directory(created_base_directory_)
, log(log_)
{
if (created_base_directory)
chassert(fs::exists(key_path));
@ -190,7 +192,7 @@ LockedKeyPtr CacheMetadata::lockKeyMetadata(
it = emplace(
key, std::make_shared<KeyMetadata>(
key, getPathForKey(key), *cleanup_queue, is_initial_load)).first;
key, getPathForKey(key), *cleanup_queue, log, is_initial_load)).first;
}
key_metadata = it->second;
@ -324,11 +326,6 @@ void CacheMetadata::doCleanup()
LockedKey::LockedKey(std::shared_ptr<KeyMetadata> key_metadata_)
: key_metadata(key_metadata_)
, lock(key_metadata->guard.lock())
#ifdef ABORT_ON_LOGICAL_ERROR
, log(&Poco::Logger::get("LockedKey(" + key_metadata_->key.toString() + ")"))
#else
, log(&Poco::Logger::get("LockedKey"))
#endif
{
}
@ -338,7 +335,7 @@ LockedKey::~LockedKey()
return;
key_metadata->key_state = KeyMetadata::KeyState::REMOVING;
LOG_DEBUG(log, "Submitting key {} for removal", getKey());
LOG_DEBUG(key_metadata->log, "Submitting key {} for removal", getKey());
key_metadata->cleanup_queue.add(getKey());
}
@ -396,20 +393,20 @@ KeyMetadata::iterator LockedKey::removeFileSegment(size_t offset, const FileSegm
auto file_segment = it->second->file_segment;
LOG_DEBUG(
log, "Remove from cache. Key: {}, offset: {}, size: {}",
key_metadata->log, "Remove from cache. Key: {}, offset: {}, size: {}",
getKey(), offset, file_segment->reserved_size);
chassert(file_segment->assertCorrectnessUnlocked(segment_lock));
if (file_segment->queue_iterator)
file_segment->queue_iterator->annul();
file_segment->queue_iterator->invalidate();
const auto path = key_metadata->getFileSegmentPath(*file_segment);
bool exists = fs::exists(path);
if (exists)
{
fs::remove(path);
LOG_TEST(log, "Removed file segment at path: {}", path);
LOG_TEST(key_metadata->log, "Removed file segment at path: {}", path);
}
else if (file_segment->downloaded_size)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected path {} to exist", path);

View File

@ -44,6 +44,7 @@ struct KeyMetadata : public std::map<size_t, FileSegmentMetadataPtr>,
const Key & key_,
const std::string & key_path_,
CleanupQueue & cleanup_queue_,
Poco::Logger * log_,
bool created_base_directory_ = false);
enum class KeyState
@ -70,6 +71,7 @@ private:
KeyGuard guard;
CleanupQueue & cleanup_queue;
std::atomic<bool> created_base_directory = false;
Poco::Logger * log;
};
using KeyMetadataPtr = std::shared_ptr<KeyMetadata>;
@ -171,7 +173,6 @@ struct LockedKey : private boost::noncopyable
private:
const std::shared_ptr<KeyMetadata> key_metadata;
KeyGuard::Lock lock; /// `lock` must be destructed before `key_metadata`.
Poco::Logger * log;
};
}

View File

@ -1476,7 +1476,7 @@ StoragePtr Context::executeTableFunction(const ASTPtr & table_expression, const
StoragePtr table = DatabaseCatalog::instance().tryGetTable({database_name, table_name}, getQueryContext());
if (table)
{
if (table.get()->isView() && table->as<StorageView>()->isParameterizedView())
if (table.get()->isView() && table->as<StorageView>() && table->as<StorageView>()->isParameterizedView())
{
function->prefer_subquery_to_function_formatting = true;
return table;

View File

@ -20,7 +20,7 @@ void OwnPatternFormatter::formatExtended(const DB::ExtendedLogMessage & msg_ext,
const Poco::Message & msg = msg_ext.base;
/// Change delimiters in date for compatibility with old logs.
DB::writeDateTimeText<'.', ':'>(msg_ext.time_seconds, wb);
DB::writeDateTimeText<'.', ':'>(msg_ext.time_seconds, wb, server_timezone);
DB::writeChar('.', wb);
DB::writeChar('0' + ((msg_ext.time_microseconds / 100000) % 10), wb);

View File

@ -2,6 +2,7 @@
#include <Poco/PatternFormatter.h>
#include <Common/DateLUT.h>
#include "ExtendedLogChannel.h"
@ -30,5 +31,6 @@ public:
virtual void formatExtended(const DB::ExtendedLogMessage & msg_ext, std::string & text) const;
private:
const DateLUTImpl & server_timezone = DateLUT::serverTimezoneInstance();
bool color;
};

View File

@ -352,7 +352,11 @@ namespace
}
if (!executor->pull(chunk))
{
if (configuration.check_exit_code)
command->wait();
return {};
}
current_read_rows += chunk.getNumRows();
}

View File

@ -33,6 +33,9 @@ struct ShellCommandSourceConfiguration
size_t number_of_rows_to_read = 0;
/// Max block size
size_t max_block_size = DEFAULT_BLOCK_SIZE;
/// Will throw if the command exited with
/// non-zero status code
size_t check_exit_code = false;
};
class ShellCommandSourceCoordinator

View File

@ -130,7 +130,7 @@ void RemoteInserter::onFinish()
break;
else if (Protocol::Server::Exception == packet.type)
packet.exception->rethrow();
else if (Protocol::Server::Log == packet.type)
else if (Protocol::Server::Log == packet.type || Protocol::Server::TimezoneUpdate == packet.type)
{
// Do nothing
}

View File

@ -512,6 +512,9 @@ RemoteQueryExecutor::ReadResult RemoteQueryExecutor::processPacket(Packet packet
throw Exception(ErrorCodes::SYSTEM_ERROR, "Could not push into profile queue");
break;
case Protocol::Server::TimezoneUpdate:
break;
default:
got_unknown_packet_from_replica = true;
throw Exception(
@ -616,6 +619,9 @@ void RemoteQueryExecutor::finish()
throw Exception(ErrorCodes::SYSTEM_ERROR, "Could not push into profile queue");
break;
case Protocol::Server::TimezoneUpdate:
break;
default:
got_unknown_packet_from_replica = true;
throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_SERVER, "Unknown packet {} from one of the following replicas: {}",

Some files were not shown because too many files have changed in this diff Show More