diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake
index 0ccd6933dec..7a9e263778e 100644
--- a/cmake/sanitize.cmake
+++ b/cmake/sanitize.cmake
@@ -57,8 +57,8 @@ if (SANITIZE)
endif ()
elseif (SANITIZE STREQUAL "undefined")
- set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero")
- set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero")
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt")
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
endif()
diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init
index f56164759bf..8f10153a682 100755
--- a/debian/clickhouse-server.init
+++ b/debian/clickhouse-server.init
@@ -153,82 +153,19 @@ initdb()
start()
{
- [ -x $CLICKHOUSE_BINDIR/$PROGRAM ] || exit 0
- local EXIT_STATUS
- EXIT_STATUS=0
-
- echo -n "Start $PROGRAM service: "
-
- if is_running; then
- echo -n "already running "
- EXIT_STATUS=1
- else
- ulimit -n 262144
- mkdir -p $CLICKHOUSE_PIDDIR
- chown -R $CLICKHOUSE_USER:$CLICKHOUSE_GROUP $CLICKHOUSE_PIDDIR
- initdb
- if ! is_running; then
- # Lock should not be held while running child process, so we release the lock. Note: obviously, there is race condition.
- # But clickhouse-server has protection from simultaneous runs with same data directory.
- su -s $SHELL ${CLICKHOUSE_USER} -c "$FLOCK -u 9; $CLICKHOUSE_PROGRAM_ENV exec -a \"$PROGRAM\" \"$CLICKHOUSE_BINDIR/$PROGRAM\" --daemon --pid-file=\"$CLICKHOUSE_PIDFILE\" --config-file=\"$CLICKHOUSE_CONFIG\""
- EXIT_STATUS=$?
- if [ $EXIT_STATUS -ne 0 ]; then
- return $EXIT_STATUS
- fi
- fi
- fi
-
- if [ $EXIT_STATUS -eq 0 ]; then
- attempts=0
- while ! is_running && [ $attempts -le ${CLICKHOUSE_START_TIMEOUT:=10} ]; do
- attempts=$(($attempts + 1))
- sleep 1
- done
- if is_running; then
- echo "DONE"
- else
- echo "UNKNOWN"
- fi
- else
- echo "FAILED"
- fi
-
- return $EXIT_STATUS
+ ${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}"
}
stop()
{
- #local EXIT_STATUS
- EXIT_STATUS=0
-
- if [ -f $CLICKHOUSE_PIDFILE ]; then
-
- echo -n "Stop $PROGRAM service: "
-
- kill -TERM $(cat "$CLICKHOUSE_PIDFILE")
-
- if ! wait_for_done ${CLICKHOUSE_STOP_TIMEOUT}; then
- EXIT_STATUS=2
- echo "TIMEOUT"
- else
- echo "DONE"
- fi
-
- fi
- return $EXIT_STATUS
+ ${CLICKHOUSE_GENERIC_PROGRAM} stop --pid-path "${CLICKHOUSE_PIDDIR}"
}
restart()
{
- check_config
- if stop; then
- if start; then
- return 0
- fi
- fi
- return 1
+ ${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}"
}
diff --git a/debian/clickhouse-server.postinst b/debian/clickhouse-server.postinst
index 6e031ae8f44..dc876f45954 100644
--- a/debian/clickhouse-server.postinst
+++ b/debian/clickhouse-server.postinst
@@ -2,6 +2,7 @@
set -e
# set -x
+PROGRAM=clickhouse-server
CLICKHOUSE_USER=${CLICKHOUSE_USER:=clickhouse}
CLICKHOUSE_GROUP=${CLICKHOUSE_GROUP:=${CLICKHOUSE_USER}}
# Please note that we don't support paths with whitespaces. This is rather ignorant.
@@ -12,6 +13,7 @@ CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR:=/usr/bin}
CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM:=clickhouse}
EXTRACT_FROM_CONFIG=${CLICKHOUSE_GENERIC_PROGRAM}-extract-from-config
CLICKHOUSE_CONFIG=$CLICKHOUSE_CONFDIR/config.xml
+CLICKHOUSE_PIDDIR=/var/run/$PROGRAM
[ -f /usr/share/debconf/confmodule ] && . /usr/share/debconf/confmodule
[ -f /etc/default/clickhouse ] && . /etc/default/clickhouse
@@ -41,105 +43,5 @@ if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then
fi
fi
- # Make sure the administrative user exists
- if ! getent passwd ${CLICKHOUSE_USER} > /dev/null; then
- if [ -n "$not_deb_os" ]; then
- useradd -r -s /bin/false --home-dir /nonexistent ${CLICKHOUSE_USER} > /dev/null
- else
- adduser --system --disabled-login --no-create-home --home /nonexistent \
- --shell /bin/false --group --gecos "ClickHouse server" ${CLICKHOUSE_USER} > /dev/null
- fi
- fi
-
- # if the user was created manually, make sure the group is there as well
- if ! getent group ${CLICKHOUSE_GROUP} > /dev/null; then
- groupadd -r ${CLICKHOUSE_GROUP} > /dev/null
- fi
-
- # make sure user is in the correct group
- if ! id -Gn ${CLICKHOUSE_USER} | grep -qw ${CLICKHOUSE_USER}; then
- usermod -a -G ${CLICKHOUSE_GROUP} ${CLICKHOUSE_USER} > /dev/null
- fi
-
- # check validity of user and group
- if [ "$(id -u ${CLICKHOUSE_USER})" -eq 0 ]; then
- echo "The ${CLICKHOUSE_USER} system user must not have uid 0 (root).
-Please fix this and reinstall this package." >&2
- exit 1
- fi
-
- if [ "$(id -g ${CLICKHOUSE_GROUP})" -eq 0 ]; then
- echo "The ${CLICKHOUSE_USER} system user must not have root as primary group.
-Please fix this and reinstall this package." >&2
- exit 1
- fi
-
- if [ -x "$CLICKHOUSE_BINDIR/$EXTRACT_FROM_CONFIG" ] && [ -f "$CLICKHOUSE_CONFIG" ]; then
- if [ -z "$SHELL" ]; then
- SHELL="/bin/sh"
- fi
- CLICKHOUSE_DATADIR_FROM_CONFIG=$(su -s $SHELL ${CLICKHOUSE_USER} -c "$CLICKHOUSE_BINDIR/$EXTRACT_FROM_CONFIG --config-file=\"$CLICKHOUSE_CONFIG\" --key=path") ||:
- echo "Path to data directory in ${CLICKHOUSE_CONFIG}: ${CLICKHOUSE_DATADIR_FROM_CONFIG}"
- fi
- CLICKHOUSE_DATADIR_FROM_CONFIG=${CLICKHOUSE_DATADIR_FROM_CONFIG:=$CLICKHOUSE_DATADIR}
-
- if [ ! -d ${CLICKHOUSE_DATADIR_FROM_CONFIG} ]; then
- mkdir -p ${CLICKHOUSE_DATADIR_FROM_CONFIG}
- chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_DATADIR_FROM_CONFIG}
- chmod 700 ${CLICKHOUSE_DATADIR_FROM_CONFIG}
- fi
-
- if [ -d ${CLICKHOUSE_CONFDIR} ]; then
- mkdir -p ${CLICKHOUSE_CONFDIR}/users.d
- mkdir -p ${CLICKHOUSE_CONFDIR}/config.d
- rm -fv ${CLICKHOUSE_CONFDIR}/*-preprocessed.xml ||:
- fi
-
- [ -e ${CLICKHOUSE_CONFDIR}/preprocessed ] || ln -s ${CLICKHOUSE_DATADIR_FROM_CONFIG}/preprocessed_configs ${CLICKHOUSE_CONFDIR}/preprocessed ||:
-
- if [ ! -d ${CLICKHOUSE_LOGDIR} ]; then
- mkdir -p ${CLICKHOUSE_LOGDIR}
- chown root:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR}
- # Allow everyone to read logs, root and clickhouse to read-write
- chmod 775 ${CLICKHOUSE_LOGDIR}
- fi
-
- # Set net_admin capabilities to support introspection of "taskstats" performance metrics from the kernel
- # and ipc_lock capabilities to allow mlock of clickhouse binary.
-
- # 1. Check that "setcap" tool exists.
- # 2. Check that an arbitrary program with installed capabilities can run.
- # 3. Set the capabilities.
-
- # The second is important for Docker and systemd-nspawn.
- # When the container has no capabilities,
- # but the executable file inside the container has capabilities,
- # then attempt to run this file will end up with a cryptic "Operation not permitted" message.
-
- TMPFILE=/tmp/test_setcap.sh
-
- command -v setcap >/dev/null \
- && echo > $TMPFILE && chmod a+x $TMPFILE && $TMPFILE && setcap "cap_net_admin,cap_ipc_lock,cap_sys_nice+ep" $TMPFILE && $TMPFILE && rm $TMPFILE \
- && setcap "cap_net_admin,cap_ipc_lock,cap_sys_nice+ep" "${CLICKHOUSE_BINDIR}/${CLICKHOUSE_GENERIC_PROGRAM}" \
- || echo "Cannot set 'net_admin' or 'ipc_lock' or 'sys_nice' capability for clickhouse binary. This is optional. Taskstats accounting will be disabled. To enable taskstats accounting you may add the required capability later manually."
-
- # Clean old dynamic compilation results
- if [ -d "${CLICKHOUSE_DATADIR_FROM_CONFIG}/build" ]; then
- rm -f ${CLICKHOUSE_DATADIR_FROM_CONFIG}/build/*.cpp ${CLICKHOUSE_DATADIR_FROM_CONFIG}/build/*.so ||:
- fi
-
- if [ -f /usr/share/debconf/confmodule ]; then
- db_get clickhouse-server/default-password
- defaultpassword="$RET"
- if [ -n "$defaultpassword" ]; then
- echo "$defaultpassword" > ${CLICKHOUSE_CONFDIR}/users.d/default-password.xml
- chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_CONFDIR}/users.d/default-password.xml
- chmod 600 ${CLICKHOUSE_CONFDIR}/users.d/default-password.xml
- fi
-
- # everything went well, so now let's reset the password
- db_set clickhouse-server/default-password ""
- # ... done with debconf here
- db_stop
- fi
+ ${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --group "${CLICKHOUSE_GROUP}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" --log-path "${CLICKHOUSE_LOGDIR}" --data-path "${CLICKHOUSE_DATADIR}"
fi
diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile
index bed438a6579..6547a98c58b 100644
--- a/docker/test/fasttest/Dockerfile
+++ b/docker/test/fasttest/Dockerfile
@@ -53,7 +53,6 @@ RUN apt-get update \
ninja-build \
psmisc \
python3 \
- python3-pip \
python3-lxml \
python3-requests \
python3-termcolor \
@@ -63,8 +62,6 @@ RUN apt-get update \
unixodbc \
--yes --no-install-recommends
-RUN pip3 install numpy scipy pandas
-
# This symlink required by gcc to find lld compiler
RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index f12ecbb2c9c..ed1346043bc 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -219,6 +219,8 @@ TESTS_TO_SKIP=(
01268_dictionary_direct_layout
01280_ssd_complex_key_dictionary
01281_group_by_limit_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently
+ 01318_encrypt # Depends on OpenSSL
+ 01318_decrypt # Depends on OpenSSL
01281_unsucceeded_insert_select_queries_counter
01292_create_user
01294_lazy_database_concurrent
diff --git a/docker/test/performance-comparison/README.md b/docker/test/performance-comparison/README.md
index 3953e99fc0f..782644a81dd 100644
--- a/docker/test/performance-comparison/README.md
+++ b/docker/test/performance-comparison/README.md
@@ -48,12 +48,13 @@ This table shows queries that take significantly longer to process on the client
#### Unexpected Query Duration
Action required for every item -- these are errors that must be fixed.
-Queries that have "short" duration (on the order of 0.1 s) can't be reliably tested in a normal way, where we perform a small (about ten) measurements for each server, because the signal-to-noise ratio is much smaller. There is a special mode for such queries that instead runs them for a fixed amount of time, normally with much higher number of measurements (up to thousands). This mode must be explicitly enabled by the test author to avoid accidental errors. It must be used only for queries that are meant to complete "immediately", such as `select count(*)`. If your query is not supposed to be "immediate", try to make it run longer, by e.g. processing more data.
+A query is supposed to run longer than 0.1 second. If your query runs faster, increase the amount of processed data to bring the run time above this threshold. You can use a bigger table (e.g. `hits_100m` instead of `hits_10m`), increase a `LIMIT`, make a query single-threaded, and so on. Queries that are too fast suffer from poor stability and precision.
-This table shows queries for which the "short" marking is not consistent with the actual query run time -- i.e., a query runs for a long time but is marked as short, or it runs very fast but is not marked as short.
+Sometimes you want to test a query that is supposed to complete "instantaneously", i.e. in sublinear time. This might be `count(*)`, or parsing a complicated tuple. It might not be practical or even possible to increase the run time of such queries by adding more data. For such queries there is a specal comparison mode which runs them for a fixed amount of time, instead of a fixed number of iterations like we do normally. This mode is inferior to the normal mode, because the influence of noise and overhead is higher, which leads to less precise and stable results.
-If your query is really supposed to complete "immediately" and can't be made to run longer, you have to mark it as "short". To do so, write `...` in the test file. The value of "short" attribute is evaluated as a python expression, and substitutions are performed, so you can write something like `select count(*) from table where {column1} > {column2}`, to mark only a particular combination of variables as short.
+If it is impossible to increase the run time of a query and it is supposed to complete "immediately", you have to explicitly mark this in the test. To do so, add a `short` attribute to the query tag in the test file: `...`. The value of the `short` attribute is evaluated as a python expression, and substitutions are performed, so you can write something like `select count(*) from table where {column1} > {column2}`, to mark only a particular combination of variables as short.
+This table shows queries for which the `short` marking is not consistent with the actual query run time -- i.e., a query runs for a normal time but is marked as `short`, or it runs faster than normal but is not marked as `short`.
#### Partial Queries
Action required for the cells marked in red.
diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py
index 1f55300661b..9d3ccabb788 100755
--- a/docker/test/performance-comparison/report.py
+++ b/docker/test/performance-comparison/report.py
@@ -468,14 +468,14 @@ if args.report == 'main':
return
columns = [
- 'Test', #0
- 'Wall clock time, s', #1
- 'Total client time, s', #2
- 'Total queries', #3
- 'Longest query (sum for all runs), s', #4
- 'Avg wall clock time (sum for all runs), s', #5
- 'Shortest query (sum for all runs), s', #6
- '', # Runs #7
+ 'Test', #0
+ 'Wall clock time, entire test, s', #1
+ 'Total client time for measured query runs, s', #2
+ 'Queries', #3
+ 'Longest query, total for measured runs, s', #4
+ 'Wall clock time per query, s', #5
+ 'Shortest query, total for measured runs, s', #6
+ '', # Runs #7
]
attrs = ['' for c in columns]
attrs[7] = None
diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md
index b85aa9b261f..c4d74ea6ee6 100644
--- a/docs/en/introduction/adopters.md
+++ b/docs/en/introduction/adopters.md
@@ -88,6 +88,7 @@ toc_title: Adopters
| SMI2 | News | Analytics | — | — | [Blog Post in Russian, November 2017](https://habr.com/ru/company/smi2/blog/314558/) |
| Splunk | Business Analytics | Main product | — | — | [Slides in English, January 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/splunk.pdf) |
| Spotify | Music | Experimentation | — | — | [Slides, July 2018](https://www.slideshare.net/glebus/using-clickhouse-for-experimentation-104247173) |
+| Staffcop | Information Security | Main Product | — | — | [Official website, Documentation](https://www.staffcop.ru/sce43) |
| Tencent | Big Data | Data processing | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) |
| Tencent | Messaging | Logging | — | — | [Talk in Chinese, November 2019](https://youtu.be/T-iVQRuw-QY?t=5050) |
| Traffic Stars | AD network | — | — | — | [Slides in Russian, May 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) |
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index f93a8b680b5..1efb7cfa8d6 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -305,6 +305,10 @@ When enabled, replace empty input fields in TSV with default values. For complex
Disabled by default.
+## input_format_tsv_enum_as_number {#settings-input_format_tsv_enum_as_number}
+
+For TSV input format switches to parsing enum values as enum ids.
+
## input_format_null_as_default {#settings-input-format-null-as-default}
Enables or disables using default values if input data contain `NULL`, but the data type of the corresponding column in not `Nullable(T)` (for text input formats).
@@ -1161,6 +1165,10 @@ The character is interpreted as a delimiter in the CSV data. By default, the del
For CSV input format enables or disables parsing of unquoted `NULL` as literal (synonym for `\N`).
+## input_format_csv_enum_as_number {#settings-input_format_csv_enum_as_number}
+
+For CSV input format switches to parsing enum values as enum ids.
+
## output_format_csv_crlf_end_of_line {#settings-output-format-csv-crlf-end-of-line}
Use DOS/Windows-style line separator (CRLF) in CSV instead of Unix style (LF).
@@ -1398,6 +1406,17 @@ Possible values:
Default value: 0
+## allow_nondeterministic_optimize_skip_unused_shards {#allow-nondeterministic-optimize-skip-unused-shards}
+
+Allow nondeterministic (like `rand` or `dictGet`, since later has some caveats with updates) functions in sharding key.
+
+Possible values:
+
+- 0 — Disallowed.
+- 1 — Allowed.
+
+Default value: 0
+
## optimize_skip_unused_shards_nesting {#optimize-skip-unused-shards-nesting}
Controls [`optimize_skip_unused_shards`](#optimize-skip-unused-shards) (hence still requires [`optimize_skip_unused_shards`](#optimize-skip-unused-shards)) depends on the nesting level of the distributed query (case when you have `Distributed` table that look into another `Distributed` table).
diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md
index 5f08417f349..881139f103c 100644
--- a/docs/en/sql-reference/functions/string-search-functions.md
+++ b/docs/en/sql-reference/functions/string-search-functions.md
@@ -461,6 +461,66 @@ For other regular expressions, the code is the same as for the ‘match’ funct
The same thing as ‘like’, but negative.
+## ilike {#ilike}
+
+Case insensitive variant of [like](https://clickhouse.tech/docs/en/sql-reference/functions/string-search-functions/#function-like) function. You can use `ILIKE` operator instead of the `ilike` function.
+
+**Syntax**
+
+``` sql
+ilike(haystack, pattern)
+```
+
+**Parameters**
+
+- `haystack` — Input string. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `pattern` — If `pattern` doesn't contain percent signs or underscores, then the `pattern` only represents the string itself. An underscore (`_`) in `pattern` stands for (matches) any single character. A percent sign (`%`) matches any sequence of zero or more characters.
+
+Some `pattern` examples:
+
+``` text
+'abc' ILIKE 'abc' true
+'abc' ILIKE 'a%' true
+'abc' ILIKE '_b_' true
+'abc' ILIKE 'c' false
+```
+
+**Returned values**
+
+- True, if the string matches `pattern`.
+- False, if the string doesn't match `pattern`.
+
+**Example**
+
+Input table:
+
+``` text
+┌─id─┬─name─────┬─days─┐
+│ 1 │ January │ 31 │
+│ 2 │ February │ 29 │
+│ 3 │ March │ 31 │
+│ 4 │ April │ 30 │
+└────┴──────────┴──────┘
+```
+
+Query:
+
+``` sql
+SELECT * FROM Months WHERE ilike(name, '%j%')
+```
+
+Result:
+
+``` text
+┌─id─┬─name────┬─days─┐
+│ 1 │ January │ 31 │
+└────┴─────────┴──────┘
+```
+
+**See Also**
+
+- [like](https://clickhouse.tech/docs/en/sql-reference/functions/string-search-functions/#function-like)
+
## ngramDistance(haystack, needle) {#ngramdistancehaystack-needle}
Calculates the 4-gram distance between `haystack` and `needle`: counts the symmetric difference between two multisets of 4-grams and normalizes it by the sum of their cardinalities. Returns float number from 0 to 1 – the closer to zero, the more strings are similar to each other. If the constant `needle` or `haystack` is more than 32Kb, throws an exception. If some of the non-constant `haystack` or `needle` strings are more than 32Kb, the distance is always one.
diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md
index e07febf9ec9..3fe3384fffc 100644
--- a/docs/en/sql-reference/operators/index.md
+++ b/docs/en/sql-reference/operators/index.md
@@ -53,6 +53,8 @@ ClickHouse transforms operators to their corresponding functions at the query pa
`a NOT LIKE s` – The `notLike(a, b)` function.
+`a ILIKE s` – The `ilike(a, b)` function.
+
`a BETWEEN b AND c` – The same as `a >= b AND a <= c`.
`a NOT BETWEEN b AND c` – The same as `a < b OR a > c`.
diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md
index 016641b958d..9aecc6c07f7 100644
--- a/docs/en/sql-reference/statements/create/table.md
+++ b/docs/en/sql-reference/statements/create/table.md
@@ -139,7 +139,7 @@ ENGINE =
```
The `Default` codec can be specified to reference default compression which may dependend on different settings (and properties of data) in runtime.
-Example: `value UInt64 CODEC(Default)` - the same as lack of codec specification.
+Example: `value UInt64 CODEC(Default)` — the same as lack of codec specification.
Also you can remove current CODEC from the column and use default compression from config.xml:
diff --git a/docs/en/sql-reference/table-functions/view.md b/docs/en/sql-reference/table-functions/view.md
new file mode 100644
index 00000000000..9997971af65
--- /dev/null
+++ b/docs/en/sql-reference/table-functions/view.md
@@ -0,0 +1,67 @@
+---
+toc_priority: 51
+toc_title: view
+---
+
+## view {#view}
+
+Turns a subquery into a table. The function implements views (see [CREATE VIEW](https://clickhouse.tech/docs/en/sql-reference/statements/create/view/#create-view)). The resulting table doesn't store data, but only stores the specified `SELECT` query. When reading from the table, ClickHouse executes the query and deletes all unnecessary columns from the result.
+
+**Syntax**
+
+``` sql
+view(subquery)
+```
+
+**Parameters**
+
+- `subquery` — `SELECT` query.
+
+**Returned value**
+
+- A table.
+
+**Example**
+
+Input table:
+
+``` text
+┌─id─┬─name─────┬─days─┐
+│ 1 │ January │ 31 │
+│ 2 │ February │ 29 │
+│ 3 │ March │ 31 │
+│ 4 │ April │ 30 │
+└────┴──────────┴──────┘
+```
+
+Query:
+
+``` sql
+SELECT * FROM view(SELECT name FROM months)
+```
+
+Result:
+
+``` text
+┌─name─────┐
+│ January │
+│ February │
+│ March │
+│ April │
+└──────────┘
+```
+
+You can use the `view` function as a parameter of the [remote](https://clickhouse.tech/docs/en/sql-reference/table-functions/remote/#remote-remotesecure) and [cluster](https://clickhouse.tech/docs/en/sql-reference/table-functions/cluster/#cluster-clusterallreplicas) table functions:
+
+``` sql
+SELECT * FROM remote(`127.0.0.1`, view(SELECT a, b, c FROM table_name))
+```
+
+``` sql
+SELECT * FROM cluster(`cluster_name`, view(SELECT a, b, c FROM table_name))
+```
+
+**See Also**
+
+- [View Table Engine](https://clickhouse.tech/docs/en/engines/table-engines/special/view/)
+[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/view/)
\ No newline at end of file
diff --git a/docs/ru/operations/system-tables/query_log.md b/docs/ru/operations/system-tables/query_log.md
index 340e77d85da..27ed93e874e 100644
--- a/docs/ru/operations/system-tables/query_log.md
+++ b/docs/ru/operations/system-tables/query_log.md
@@ -34,6 +34,7 @@ ClickHouse не удаляет данные из таблица автомати
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата начала запроса.
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала запроса.
- `query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала обработки запроса.
+- `query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время начала обработки запроса с точностью до микросекунд.
- `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — длительность выполнения запроса в миллисекундах.
- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Общее количество строк, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_rows` включает в себя общее количество строк, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_rows`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся.
- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Общее количество байтов, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_bytes` включает в себя общее количество байтов, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_bytes`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся.
diff --git a/docs/ru/operations/system-tables/query_thread_log.md b/docs/ru/operations/system-tables/query_thread_log.md
index acee3063323..11571f594d5 100644
--- a/docs/ru/operations/system-tables/query_thread_log.md
+++ b/docs/ru/operations/system-tables/query_thread_log.md
@@ -16,6 +16,7 @@ ClickHouse не удаляет данные из таблицы автомати
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата завершения выполнения запроса потоком.
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — дата и время завершения выполнения запроса потоком.
- `query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала обработки запроса.
+- `query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время начала обработки запроса с точностью до микросекунд.
- `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — длительность обработки запроса в миллисекундах.
- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — количество прочитанных строк.
- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — количество прочитанных байтов.
diff --git a/docs/ru/sql-reference/functions/string-search-functions.md b/docs/ru/sql-reference/functions/string-search-functions.md
index 29dd67fd0eb..078a09a8aa4 100644
--- a/docs/ru/sql-reference/functions/string-search-functions.md
+++ b/docs/ru/sql-reference/functions/string-search-functions.md
@@ -442,6 +442,66 @@ SELECT extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[
То же, что like, но с отрицанием.
+## ilike {#ilike}
+
+Нечувствительный к регистру вариант функции [like](https://clickhouse.tech/docs/ru/sql-reference/functions/string-search-functions/#function-like). Вы можете использовать оператор `ILIKE` вместо функции `ilike`.
+
+**Синтаксис**
+
+``` sql
+ilike(haystack, pattern)
+```
+
+**Параметры**
+
+- `haystack` — Входная строка. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `pattern` — Если `pattern` не содержит процента или нижнего подчеркивания, тогда `pattern` представляет саму строку. Нижнее подчеркивание (`_`) в `pattern` обозначает любой отдельный символ. Знак процента (`%`) соответствует последовательности из любого количества символов: от нуля и более.
+
+Некоторые примеры `pattern`:
+
+``` text
+'abc' ILIKE 'abc' true
+'abc' ILIKE 'a%' true
+'abc' ILIKE '_b_' true
+'abc' ILIKE 'c' false
+```
+
+**Возвращаемые значения**
+
+- Правда, если строка соответствует `pattern`.
+- Ложь, если строка не соответствует `pattern`.
+
+**Пример**
+
+Входная таблица:
+
+``` text
+┌─id─┬─name─────┬─days─┐
+│ 1 │ January │ 31 │
+│ 2 │ February │ 29 │
+│ 3 │ March │ 31 │
+│ 4 │ April │ 30 │
+└────┴──────────┴──────┘
+```
+
+Запрос:
+
+``` sql
+SELECT * FROM Months WHERE ilike(name, '%j%')
+```
+
+Результат:
+
+``` text
+┌─id─┬─name────┬─days─┐
+│ 1 │ January │ 31 │
+└────┴─────────┴──────┘
+```
+
+**Смотрите также**
+
+- [like](https://clickhouse.tech/docs/ru/sql-reference/functions/string-search-functions/#function-like)
+
## ngramDistance(haystack, needle) {#ngramdistancehaystack-needle}
Вычисление 4-граммного расстояния между `haystack` и `needle`: считается симметрическая разность между двумя мультимножествами 4-грамм и нормализуется на сумму их мощностей. Возвращает число float от 0 до 1 – чем ближе к нулю, тем больше строки похожи друг на друга. Если константный `needle` или `haystack` больше чем 32КБ, кидается исключение. Если некоторые строки из неконстантного `haystack` или `needle` больше 32КБ, расстояние всегда равно единице.
diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md
index ef4d1ae048e..08594193d4c 100644
--- a/docs/ru/sql-reference/operators/index.md
+++ b/docs/ru/sql-reference/operators/index.md
@@ -49,6 +49,8 @@
`a NOT LIKE s` - функция `notLike(a, b)`
+`a ILIKE s` – функция `ilike(a, b)`
+
`a BETWEEN b AND c` - равнозначно `a >= b AND a <= c`
`a NOT BETWEEN b AND c` - равнозначно `a < b OR a > c`
diff --git a/docs/ru/sql-reference/statements/create/table.md b/docs/ru/sql-reference/statements/create/table.md
index 74eb0ff56e7..c6093bbc6de 100644
--- a/docs/ru/sql-reference/statements/create/table.md
+++ b/docs/ru/sql-reference/statements/create/table.md
@@ -119,7 +119,18 @@ ENGINE =
...
```
-Если задать кодек для столбца, то кодек по умолчанию не применяется. Кодеки можно последовательно комбинировать, например, `CODEC(Delta, ZSTD)`. Чтобы выбрать наиболее подходящую для вашего проекта комбинацию кодеков, необходимо провести сравнительные тесты, подобные тем, что описаны в статье Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse).
+Если кодек `Default` задан для столбца, используется сжатие по умолчанию, которое может зависеть от различных настроек (и свойств данных) во время выполнения.
+Пример: `value UInt64 CODEC(Default)` — то же самое, что не указать кодек.
+
+Также можно подменить кодек столбца сжатием по умолчанию, определенным в config.xml:
+
+``` sql
+ALTER TABLE codec_example MODIFY COLUMN float_value CODEC(Default);
+```
+
+Кодеки можно последовательно комбинировать, например, `CODEC(Delta, Default)`.
+
+Чтобы выбрать наиболее подходящую для вашего проекта комбинацию кодеков, необходимо провести сравнительные тесты, подобные тем, что описаны в статье Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse). Для столбцов типа `ALIAS` кодеки не применяются.
!!! warning "Предупреждение"
Нельзя распаковать базу данных ClickHouse с помощью сторонних утилит наподобие `lz4`. Необходимо использовать специальную утилиту [clickhouse-compressor](https://github.com/ClickHouse/ClickHouse/tree/master/programs/compressor).
@@ -195,4 +206,4 @@ CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name
[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/table)
-
\ No newline at end of file
+
diff --git a/docs/ru/sql-reference/table-functions/view.md b/docs/ru/sql-reference/table-functions/view.md
new file mode 100644
index 00000000000..8a97253d048
--- /dev/null
+++ b/docs/ru/sql-reference/table-functions/view.md
@@ -0,0 +1,62 @@
+## view {#view}
+
+Преобразовывает подзапрос в таблицу. Функция реализовывает представления (смотрите [CREATE VIEW](https://clickhouse.tech/docs/ru/sql-reference/statements/create/view/#create-view)). Результирующая таблица не хранит данные, а только сохраняет указанный запрос `SELECT`. При чтении из таблицы, ClickHouse выполняет запрос и удаляет все ненужные столбцы из результата.
+
+**Синтаксис**
+
+``` sql
+view(subquery)
+```
+
+**Входные параметры**
+
+- `subquery` — запрос `SELECT`.
+
+**Возвращаемое значение**
+
+- Таблица.
+
+**Пример**
+
+Входная таблица:
+
+``` text
+┌─id─┬─name─────┬─days─┐
+│ 1 │ January │ 31 │
+│ 2 │ February │ 29 │
+│ 3 │ March │ 31 │
+│ 4 │ April │ 30 │
+└────┴──────────┴──────┘
+```
+
+Запрос:
+
+``` sql
+SELECT * FROM view(SELECT name FROM months)
+```
+
+Результат:
+
+``` text
+┌─name─────┐
+│ January │
+│ February │
+│ March │
+│ April │
+└──────────┘
+```
+
+Вы можете использовать функцию `view` как параметр табличных функций [remote](https://clickhouse.tech/docs/ru/sql-reference/table-functions/remote/#remote-remotesecure) и [cluster](https://clickhouse.tech/docs/ru/sql-reference/table-functions/cluster/#cluster-clusterallreplicas):
+
+``` sql
+SELECT * FROM remote(`127.0.0.1`, view(SELECT a, b, c FROM table_name))
+```
+
+``` sql
+SELECT * FROM cluster(`cluster_name`, view(SELECT a, b, c FROM table_name))
+```
+
+**Смотрите также**
+
+- [view](https://clickhouse.tech/docs/ru/engines/table-engines/special/view/#table_engines-view)
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/view/)
\ No newline at end of file
diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt
index 68a72a6a785..c4559696b16 100644
--- a/docs/tools/requirements.txt
+++ b/docs/tools/requirements.txt
@@ -14,7 +14,7 @@ Jinja2==2.11.2
jinja2-highlight==0.6.1
jsmin==2.2.2
livereload==2.6.2
-Markdown==3.2.1
+Markdown==3.3.2
MarkupSafe==1.1.1
mkdocs==1.1.2
mkdocs-htmlproofer-plugin==0.0.3
diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp
index ae0c22c8fcc..8290118089c 100644
--- a/programs/install/Install.cpp
+++ b/programs/install/Install.cpp
@@ -548,11 +548,27 @@ int mainEntryClickHouseInstall(int argc, char ** argv)
users_config_file.string(), users_d.string());
}
- /// Set capabilities for the binary.
+ /** Set capabilities for the binary.
+ *
+ * 1. Check that "setcap" tool exists.
+ * 2. Check that an arbitrary program with installed capabilities can run.
+ * 3. Set the capabilities.
+ *
+ * The second is important for Docker and systemd-nspawn.
+ * When the container has no capabilities,
+ * but the executable file inside the container has capabilities,
+ * then attempt to run this file will end up with a cryptic "Operation not permitted" message.
+ */
#if defined(__linux__)
fmt::print("Setting capabilities for clickhouse binary. This is optional.\n");
- std::string command = fmt::format("command -v setcap && setcap 'cap_net_admin,cap_ipc_lock,cap_sys_nice+ep' {}", main_bin_path.string());
+ std::string command = fmt::format("command -v setcap >/dev/null"
+ " && echo > {0} && chmod a+x {0} && {0} && setcap 'cap_net_admin,cap_ipc_lock,cap_sys_nice+ep' {0} && {0} && rm {0}"
+ " && setcap 'cap_net_admin,cap_ipc_lock,cap_sys_nice+ep' {1}"
+ " || echo \"Cannot set 'net_admin' or 'ipc_lock' or 'sys_nice' capability for clickhouse binary."
+ " This is optional. Taskstats accounting will be disabled."
+ " To enable taskstats accounting you may add the required capability later manually.\"",
+ "/tmp/test_setcap.sh", main_bin_path.string());
fmt::print(" {}\n", command);
executeScript(command);
#endif
diff --git a/programs/server/CMakeLists.txt b/programs/server/CMakeLists.txt
index b3dcf1955fe..198d9081168 100644
--- a/programs/server/CMakeLists.txt
+++ b/programs/server/CMakeLists.txt
@@ -4,7 +4,7 @@ set(CLICKHOUSE_SERVER_SOURCES
)
if (OS_LINUX)
- set (LINK_CONFIG_LIB INTERFACE "-Wl,${WHOLE_ARCHIVE} $ -Wl,${NO_WHOLE_ARCHIVE}")
+ set (LINK_RESOURCE_LIB INTERFACE "-Wl,${WHOLE_ARCHIVE} $ -Wl,${NO_WHOLE_ARCHIVE}")
endif ()
set (CLICKHOUSE_SERVER_LINK
@@ -20,7 +20,7 @@ set (CLICKHOUSE_SERVER_LINK
clickhouse_table_functions
string_utils
- ${LINK_CONFIG_LIB}
+ ${LINK_RESOURCE_LIB}
PUBLIC
daemon
@@ -37,20 +37,20 @@ if (OS_LINUX)
# 1. Allow to run the binary without download of any other files.
# 2. Allow to implement "sudo clickhouse install" tool.
- foreach(CONFIG_FILE config users embedded)
- set(CONFIG_OBJ ${CONFIG_FILE}.o)
- set(CONFIG_OBJS ${CONFIG_OBJS} ${CONFIG_OBJ})
+ foreach(RESOURCE_FILE config.xml users.xml embedded.xml play.html)
+ set(RESOURCE_OBJ ${RESOURCE_FILE}.o)
+ set(RESOURCE_OBJS ${RESOURCE_OBJS} ${RESOURCE_OBJ})
# https://stackoverflow.com/questions/14776463/compile-and-add-an-object-file-from-a-binary-with-cmake
- add_custom_command(OUTPUT ${CONFIG_OBJ}
- COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${OBJCOPY_PATH} -I binary ${OBJCOPY_ARCH_OPTIONS} ${CONFIG_FILE}.xml ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_OBJ}
+ add_custom_command(OUTPUT ${RESOURCE_OBJ}
+ COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${OBJCOPY_PATH} -I binary ${OBJCOPY_ARCH_OPTIONS} ${RESOURCE_FILE} ${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ}
COMMAND ${OBJCOPY_PATH} --rename-section .data=.rodata,alloc,load,readonly,data,contents
- ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_OBJ} ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_OBJ})
+ ${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ} ${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ})
- set_source_files_properties(${CONFIG_OBJ} PROPERTIES EXTERNAL_OBJECT true GENERATED true)
- endforeach(CONFIG_FILE)
+ set_source_files_properties(${RESOURCE_OBJ} PROPERTIES EXTERNAL_OBJECT true GENERATED true)
+ endforeach(RESOURCE_FILE)
- add_library(clickhouse_server_configs STATIC ${CONFIG_OBJS})
+ add_library(clickhouse_server_configs STATIC ${RESOURCE_OBJS})
set_target_properties(clickhouse_server_configs PROPERTIES LINKER_LANGUAGE C)
# whole-archive prevents symbols from being discarded for unknown reason
diff --git a/programs/server/config.xml b/programs/server/config.xml
index 83921b12382..9850d77abb7 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -212,22 +212,10 @@
/var/lib/clickhouse/user_files/
-
-
-
-
- users.xml
-
-
-
- /var/lib/clickhouse/access/
-
-
-
-
+
-
+
+
+
+
+ users.xml
+
+
+
+ /var/lib/clickhouse/access/
+
+
+
+
+
default
@@ -704,18 +722,22 @@
-->
/var/lib/clickhouse/format_schemas/
-
- hide SSN
- \b\d{3}-\d{2}-\d{4}\b
- 000-00-0000
+ hide encrypt/decrypt arguments
+ ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\)
+
+ \1(???)
- -->
+
+
+ ClickHouse Query
+
+
+
+
+
+
+
+