diff --git a/.pylintrc b/.pylintrc index add9a2aa94b..b672cbfdfad 100644 --- a/.pylintrc +++ b/.pylintrc @@ -13,9 +13,7 @@ max-statements=200 ignore-long-lines = (# )??$ [MESSAGES CONTROL] -disable = bad-continuation, - missing-docstring, - bad-whitespace, +disable = missing-docstring, too-few-public-methods, invalid-name, too-many-arguments, diff --git a/docker/docs/builder/Dockerfile b/docker/docs/builder/Dockerfile index a3b5bffdfe8..76bbea59e60 100644 --- a/docker/docs/builder/Dockerfile +++ b/docker/docs/builder/Dockerfile @@ -1,15 +1,16 @@ # rebuild in #36968 # docker build -t clickhouse/docs-builder . # nodejs 17 prefers ipv6 and is broken in our environment -FROM node:16.14.2-alpine3.15 +FROM node:16-alpine RUN apk add --no-cache git openssh bash -# TODO: clean before merge! -ARG DOCS_BRANCH=main +# At this point we want to really update /opt/clickhouse-docs +# despite the cached images +ARG CACHE_INVALIDATOR=0 RUN git clone https://github.com/ClickHouse/clickhouse-docs.git \ - --depth=1 --branch=${DOCS_BRANCH} /opt/clickhouse-docs + --depth=1 --branch=main /opt/clickhouse-docs WORKDIR /opt/clickhouse-docs diff --git a/docker/docs/builder/run.sh b/docker/docs/builder/run.sh index 2e13bfb55bb..a4f678b2f24 100755 --- a/docker/docs/builder/run.sh +++ b/docker/docs/builder/run.sh @@ -8,8 +8,6 @@ if [ "$GIT_DOCS_BRANCH" ] && ! [ "$GIT_DOCS_BRANCH" == "$GIT_BRANCH" ]; then git fetch origin --depth=1 -- "$GIT_DOCS_BRANCH:$GIT_DOCS_BRANCH" git checkout "$GIT_DOCS_BRANCH" else - # Untracked yarn.lock could cause pull to fail - git clean -fdx # Update docs repo git pull fi diff --git a/docker/keeper/entrypoint.sh b/docker/keeper/entrypoint.sh index 3aacf655c28..86e56e88aa9 100644 --- a/docker/keeper/entrypoint.sh +++ b/docker/keeper/entrypoint.sh @@ -42,6 +42,7 @@ DATA_DIR="${CLICKHOUSE_DATA_DIR:-/var/lib/clickhouse}" LOG_DIR="${LOG_DIR:-/var/log/clickhouse-keeper}" LOG_PATH="${LOG_DIR}/clickhouse-keeper.log" ERROR_LOG_PATH="${LOG_DIR}/clickhouse-keeper.err.log" +COORDINATION_DIR="${DATA_DIR}/coordination" COORDINATION_LOG_DIR="${DATA_DIR}/coordination/log" COORDINATION_SNAPSHOT_DIR="${DATA_DIR}/coordination/snapshots" CLICKHOUSE_WATCHDOG_ENABLE=${CLICKHOUSE_WATCHDOG_ENABLE:-0} @@ -49,6 +50,7 @@ CLICKHOUSE_WATCHDOG_ENABLE=${CLICKHOUSE_WATCHDOG_ENABLE:-0} for dir in "$DATA_DIR" \ "$LOG_DIR" \ "$TMP_DIR" \ + "$COORDINATION_DIR" \ "$COORDINATION_LOG_DIR" \ "$COORDINATION_SNAPSHOT_DIR" do diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index 7d919372c2b..c2ed47a1392 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -8,17 +8,16 @@ ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ + aspell \ curl \ git \ libxml2-utils \ moreutils \ - pylint \ python3-fuzzywuzzy \ python3-pip \ shellcheck \ yamllint \ - aspell \ - && pip3 install black boto3 codespell dohq-artifactory PyGithub unidiff + && pip3 install black boto3 codespell dohq-artifactory PyGithub unidiff pylint==2.6.2 # Architecture of the image when BuildKit/buildx is used ARG TARGETARCH diff --git a/docs/en/sql-reference/functions/index.md b/docs/en/sql-reference/functions/index.md index 4574d78bc29..2dadf91e069 100644 --- a/docs/en/sql-reference/functions/index.md +++ b/docs/en/sql-reference/functions/index.md @@ -174,22 +174,24 @@ Result: Creating `test_function_sum_json` with named arguments and format [JSONEachRow](../../interfaces/formats.md#jsoneachrow) using XML configuration. File test_function.xml. ```xml - - executable - test_function_sum_json - UInt64 - result_name - - UInt64 - argument_1 - - - UInt64 - argument_2 - - JSONEachRow - test_function_sum_json.py - + + + executable + test_function_sum_json + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + argument_2 + + JSONEachRow + test_function_sum_json.py + + ``` Script file inside `user_scripts` folder `test_function_sum_json.py`. @@ -224,6 +226,50 @@ Result: └──────────────────────────────┘ ``` +Executable user defined functions can take constant parameters configured in `command` setting (works only for user defined functions with `executable` type). +File test_function_parameter_python.xml. +```xml + + + executable + test_function_parameter_python + String + + UInt64 + + TabSeparated + test_function_parameter_python.py {test_parameter:UInt64} + + +``` + +Script file inside `user_scripts` folder `test_function_parameter_python.py`. + +```python +#!/usr/bin/python3 + +import sys + +if __name__ == "__main__": + for line in sys.stdin: + print("Parameter " + str(sys.argv[1]) + " value " + str(line), end="") + sys.stdout.flush() +``` + +Query: + +``` sql +SELECT test_function_parameter_python(1)(2); +``` + +Result: + +``` text +┌─test_function_parameter_python(1)(2)─┐ +│ Parameter 1 value 2 │ +└──────────────────────────────────────┘ +``` + ## Error Handling Some functions might throw an exception if the data is invalid. In this case, the query is canceled and an error text is returned to the client. For distributed processing, when an exception occurs on one of the servers, the other servers also attempt to abort the query. diff --git a/docs/ru/sql-reference/functions/index.md b/docs/ru/sql-reference/functions/index.md index fc50e57cc72..9bde1f2e56c 100644 --- a/docs/ru/sql-reference/functions/index.md +++ b/docs/ru/sql-reference/functions/index.md @@ -174,22 +174,24 @@ SELECT test_function_sum(2, 2); Создание `test_function_sum_json` с именноваными аргументами и форматом [JSONEachRow](../../interfaces/formats.md#jsoneachrow) с использованием конфигурации XML. Файл test_function.xml. ```xml - - executable - test_function_sum_json - UInt64 - result_name - - UInt64 - argument_1 - - - UInt64 - argument_2 - - JSONEachRow - test_function_sum_json.py - + + + executable + test_function_sum_json + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + argument_2 + + JSONEachRow + test_function_sum_json.py + + ``` Файл скрипта внутри папки `user_scripts` `test_function_sum_json.py`. @@ -224,6 +226,50 @@ SELECT test_function_sum_json(2, 2); └──────────────────────────────┘ ``` +Исполняемые пользовательские функции могут принимать константные параметры, их конфигурация является частью настройки `command` (работает только для пользовательских функций с типом `executable`). +Файл test_function_parameter_python.xml. +```xml + + + executable + test_function_parameter_python + String + + UInt64 + + TabSeparated + test_function_parameter_python.py {test_parameter:UInt64} + + +``` + +Файл скрипта внутри папки `user_scripts` `test_function_parameter_python.py`. + +```python +#!/usr/bin/python3 + +import sys + +if __name__ == "__main__": + for line in sys.stdin: + print("Parameter " + str(sys.argv[1]) + " value " + str(line), end="") + sys.stdout.flush() +``` + +Query: + +``` sql +SELECT test_function_parameter_python(1)(2); +``` + +Result: + +``` text +┌─test_function_parameter_python(1)(2)─┐ +│ Parameter 1 value 2 │ +└──────────────────────────────────────┘ +``` + ## Обработка ошибок {#obrabotka-oshibok} Некоторые функции могут кидать исключения в случае ошибочных данных. В этом случае, выполнение запроса прерывается, и текст ошибки выводится клиенту. При распределённой обработке запроса, при возникновении исключения на одном из серверов, на другие серверы пытается отправиться просьба тоже прервать выполнение запроса. diff --git a/src/Common/ProcfsMetricsProvider.h b/src/Common/ProcfsMetricsProvider.h index 785ee077fdc..5ca47b78e44 100644 --- a/src/Common/ProcfsMetricsProvider.h +++ b/src/Common/ProcfsMetricsProvider.h @@ -19,7 +19,7 @@ public: /// Updates only a part of taskstats struct's fields: /// - cpu_run_virtual_total, cpu_delay_total (when /proc/thread-self/schedstat is available) /// - blkio_delay_total (when /proc/thread-self/stat is available) - /// - rchar, wchar, read_bytes, write_bytes (when /prod/thread-self/io is available) + /// - rchar, wchar, read_bytes, write_bytes (when /proc/thread-self/io is available) /// See: man procfs void getTaskStats(::taskstats & out_stats) const; diff --git a/src/Functions/CastOverloadResolver.h b/src/Functions/CastOverloadResolver.h index 9b579c34923..e62020e10fe 100644 --- a/src/Functions/CastOverloadResolver.h +++ b/src/Functions/CastOverloadResolver.h @@ -45,7 +45,7 @@ public: const auto & settings_ref = context->getSettingsRef(); if constexpr (internal) - return createImpl({}, false /*keep_nullable*/, false /*cast_ipv4_ipv6_default_on_conversion_error*/); + return createImpl({}, false /*keep_nullable*/, settings_ref.cast_ipv4_ipv6_default_on_conversion_error); return createImpl({}, settings_ref.cast_keep_nullable, settings_ref.cast_ipv4_ipv6_default_on_conversion_error); } diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index 6435948b870..4bc7a17cf4c 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -458,6 +458,8 @@ public: { default_cols.emplace_back(result); } + + ++current_arguments_index; } else { @@ -465,6 +467,13 @@ public: default_cols.emplace_back(nullptr); } + if (current_arguments_index < arguments.size()) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {} should be {}", + getName(), + arguments.size(), + current_arguments_index); + auto key_col_with_type = arguments[2]; bool key_is_only_null = key_col_with_type.type->onlyNull(); diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index c1d1c1df1f1..a0d0e124f6d 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -12,14 +12,14 @@ from typing import Dict, List, Optional, Set, Tuple, Union from github import Github -from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, GITHUB_RUN_URL -from s3_helper import S3Helper -from pr_info import PRInfo -from get_robot_token import get_best_robot_token, get_parameter_from_ssm -from upload_result_helper import upload_results -from commit_status_helper import post_commit_status from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse +from commit_status_helper import post_commit_status +from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, GITHUB_RUN_URL +from get_robot_token import get_best_robot_token, get_parameter_from_ssm +from pr_info import PRInfo +from s3_helper import S3Helper from stopwatch import Stopwatch +from upload_result_helper import upload_results NAME = "Push to Dockerhub (actions)" @@ -237,6 +237,8 @@ def build_and_push_one_image( "docker buildx build --builder default " f"--label build-url={GITHUB_RUN_URL} " f"{from_tag_arg}" + # A hack to invalidate cache, grep for it in docker/ dir + f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " f"--tag {image.repo}:{version_string} " f"--cache-from type=registry,ref={image.repo}:{version_string} " f"--cache-from type=registry,ref={image.repo}:latest " diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py index cabf12082e8..550d495939c 100644 --- a/tests/ci/docker_test.py +++ b/tests/ci/docker_test.py @@ -124,6 +124,7 @@ class TestDockerImageCheck(unittest.TestCase): self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " "--build-arg FROM_TAG=version " + f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version --cache-from type=registry,ref=name:version " "--cache-from type=registry,ref=name:latest " "--cache-to type=inline,mode=max --push --progress plain path", @@ -142,6 +143,7 @@ class TestDockerImageCheck(unittest.TestCase): self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " "--build-arg FROM_TAG=version2 " + f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version2 --cache-from type=registry,ref=name:version2 " "--cache-from type=registry,ref=name:latest " "--cache-to type=inline,mode=max --progress plain path", @@ -159,6 +161,7 @@ class TestDockerImageCheck(unittest.TestCase): mock_machine.assert_not_called() self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " + f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version2 --cache-from type=registry,ref=name:version2 " "--cache-from type=registry,ref=name:latest " "--cache-to type=inline,mode=max --progress plain path", diff --git a/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.reference b/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql b/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql new file mode 100644 index 00000000000..e8071877379 --- /dev/null +++ b/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql @@ -0,0 +1,24 @@ +SET cast_ipv4_ipv6_default_on_conversion_error = 1; + +DROP TABLE IF EXISTS ipv4_test; +CREATE TABLE ipv4_test +( + id UInt64, + value String +) ENGINE=MergeTree ORDER BY id; + +ALTER TABLE ipv4_test MODIFY COLUMN value IPv4 DEFAULT ''; + +DROP TABLE ipv4_test; + +DROP TABLE IF EXISTS ipv6_test; +CREATE TABLE ipv6_test +( + id UInt64, + value String +) ENGINE=MergeTree ORDER BY id; + +ALTER TABLE ipv6_test MODIFY COLUMN value IPv4 DEFAULT ''; +SELECT * FROM ipv6_test; + +DROP TABLE ipv6_test; diff --git a/tests/queries/0_stateless/02319_dict_get_check_arguments_size.reference b/tests/queries/0_stateless/02319_dict_get_check_arguments_size.reference new file mode 100644 index 00000000000..7c7157fb549 --- /dev/null +++ b/tests/queries/0_stateless/02319_dict_get_check_arguments_size.reference @@ -0,0 +1,4 @@ +Value +DefaultValue +Value +DefaultValue diff --git a/tests/queries/0_stateless/02319_dict_get_check_arguments_size.sql b/tests/queries/0_stateless/02319_dict_get_check_arguments_size.sql new file mode 100644 index 00000000000..089d783a123 --- /dev/null +++ b/tests/queries/0_stateless/02319_dict_get_check_arguments_size.sql @@ -0,0 +1,59 @@ +DROP TABLE IF EXISTS dictionary_source_table; +CREATE TABLE dictionary_source_table +( + id UInt64, + value String +) ENGINE=TinyLog; + +INSERT INTO dictionary_source_table VALUES (0, 'Value'); + +DROP DICTIONARY IF EXISTS test_dictionary; +CREATE DICTIONARY test_dictionary +( + id UInt64, + value String +) +PRIMARY KEY id +LAYOUT(FLAT()) +SOURCE(CLICKHOUSE(TABLE 'dictionary_source_table')) +LIFETIME(0); + +SELECT dictGet('test_dictionary', 'value', 0); +SELECT dictGet('test_dictionary', 'value', 0, 'DefaultValue'); --{serverError 42} +SELECT dictGetOrDefault('test_dictionary', 'value', 1, 'DefaultValue'); +SELECT dictGetOrDefault('test_dictionary', 'value', 1, 'DefaultValue', 1); --{serverError 42} + +DROP DICTIONARY test_dictionary; + +DROP TABLE dictionary_source_table; +CREATE TABLE dictionary_source_table +( + key UInt64, + start UInt64, + end UInt64, + value String +) Engine = TinyLog; + +INSERT INTO dictionary_source_table values (0, 0, 5, 'Value'); + +DROP DICTIONARY IF EXISTS range_hashed_dictionary; +CREATE DICTIONARY range_hashed_dictionary +( + key UInt64, + start UInt64, + end UInt64, + value String +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(TABLE 'dictionary_source_table')) +LAYOUT(RANGE_HASHED()) +RANGE(MIN start MAX end) +LIFETIME(0); + +SELECT dictGet('range_hashed_dictionary', 'value', 0, toUInt64(4)); +SELECT dictGet('range_hashed_dictionary', 'value', 4, toUInt64(6), 'DefaultValue'); --{serverError 42} +SELECT dictGetOrDefault('range_hashed_dictionary', 'value', 1, toUInt64(6), 'DefaultValue'); +SELECT dictGetOrDefault('range_hashed_dictionary', 'value', 1, toUInt64(6), 'DefaultValue', 1); --{serverError 42} + +DROP DICTIONARY range_hashed_dictionary; +DROP TABLE dictionary_source_table;