From d10c32ec6715a0aea6d35b15a7726231b0cd8cf6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 14 Jan 2022 22:02:58 +0300 Subject: [PATCH 0001/1954] Enable some settings --- src/Core/Settings.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index d55be808aa8..5d19de73e74 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -120,7 +120,7 @@ class IColumn; M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \ M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \ M(UInt64, aggregation_memory_efficient_merge_threads, 0, "Number of threads to use for merge intermediate aggregation results in memory efficient mode. When bigger, then more memory is consumed. 0 means - same as 'max_threads'.", 0) \ - M(Bool, enable_positional_arguments, false, "Enable positional arguments in ORDER BY, GROUP BY and LIMIT BY", 0) \ + M(Bool, enable_positional_arguments, true, "Enable positional arguments in ORDER BY, GROUP BY and LIMIT BY", 0) \ \ M(UInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled.", 0) \ M(UInt64, parallel_replicas_count, 0, "", 0) \ @@ -437,7 +437,7 @@ class IColumn; M(Bool, optimize_if_chain_to_multiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \ M(Bool, optimize_if_transform_strings_to_enum, false, "Replaces string-type arguments in If and Transform to enum. Disabled by default cause it could make inconsistent change in distributed query that would lead to its fail.", 0) \ M(Bool, optimize_monotonous_functions_in_order_by, true, "Replace monotonous function with its argument in ORDER BY", 0) \ - M(Bool, optimize_functions_to_subcolumns, false, "Transform functions to subcolumns, if possible, to reduce amount of read data. E.g. 'length(arr)' -> 'arr.size0', 'col IS NULL' -> 'col.null' ", 0) \ + M(Bool, optimize_functions_to_subcolumns, true, "Transform functions to subcolumns, if possible, to reduce amount of read data. E.g. 'length(arr)' -> 'arr.size0', 'col IS NULL' -> 'col.null' ", 0) \ M(Bool, optimize_using_constraints, false, "Use constraints for query optimization", 0) \ M(Bool, optimize_substitute_columns, false, "Use constraints for column substitution", 0) \ M(Bool, optimize_append_index, false, "Use constraints in order to append index condition (indexHint)", 0) \ @@ -455,7 +455,7 @@ class IColumn; M(Seconds, lock_acquire_timeout, DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC, "How long locking request should wait before failing", 0) \ M(Bool, materialize_ttl_after_modify, true, "Apply TTL for old data, after ALTER MODIFY TTL query", 0) \ M(String, function_implementation, "", "Choose function implementation for specific target or variant (experimental). If empty enable all of them.", 0) \ - M(Bool, allow_experimental_geo_types, false, "Allow geo data types such as Point, Ring, Polygon, MultiPolygon", 0) \ + M(Bool, allow_experimental_geo_types, true, "Allow geo data types such as Point, Ring, Polygon, MultiPolygon", 0) \ M(Bool, data_type_default_nullable, false, "Data types without NULL or NOT NULL will make Nullable", 0) \ M(Bool, cast_keep_nullable, false, "CAST operator keep Nullable for result data type", 0) \ M(Bool, alter_partition_verbose_result, false, "Output information about affected parts. Currently works only for FREEZE and ATTACH commands.", 0) \ @@ -467,8 +467,8 @@ class IColumn; M(Bool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \ M(Bool, enable_global_with_statement, true, "Propagate WITH statements to UNION queries and all subqueries", 0) \ M(Bool, aggregate_functions_null_for_empty, false, "Rewrite all aggregate functions in a query, adding -OrNull suffix to them", 0) \ - M(Bool, optimize_syntax_fuse_functions, false, "Allow apply syntax optimisation: fuse aggregate functions", 0) \ - M(Bool, optimize_fuse_sum_count_avg, false, "Fuse functions `sum, avg, count` with identical arguments into one `sumCount` (`optimize_syntax_fuse_functions should be enabled)", 0) \ + M(Bool, optimize_syntax_fuse_functions, true, "Allow apply syntax optimisation: fuse aggregate functions", 0) \ + M(Bool, optimize_fuse_sum_count_avg, true, "Fuse functions `sum, avg, count` with identical arguments into one `sumCount` (`optimize_syntax_fuse_functions should be enabled)", 0) \ M(Bool, flatten_nested, true, "If true, columns of type Nested will be flatten to separate array columns instead of one array of tuples", 0) \ M(Bool, asterisk_include_materialized_columns, false, "Include MATERIALIZED columns for wildcard query", 0) \ M(Bool, asterisk_include_alias_columns, false, "Include ALIAS columns for wildcard query", 0) \ @@ -518,7 +518,7 @@ class IColumn; M(UInt64, function_range_max_elements_in_block, 500000000, "Maximum number of values generated by function 'range' per block of data (sum of array sizes for every row in a block, see also 'max_block_size' and 'min_insert_block_size_rows'). It is a safety threshold.", 0) \ M(ShortCircuitFunctionEvaluation, short_circuit_function_evaluation, ShortCircuitFunctionEvaluation::ENABLE, "Setting for short-circuit function evaluation configuration. Possible values: 'enable' - use short-circuit function evaluation for functions that are suitable for it, 'disable' - disable short-circuit function evaluation, 'force_enable' - use short-circuit function evaluation for all functions.", 0) \ \ - M(String, local_filesystem_read_method, "pread", "Method of reading data from local filesystem, one of: read, pread, mmap, pread_threadpool.", 0) \ + M(String, local_filesystem_read_method, "pread_threadpool", "Method of reading data from local filesystem, one of: read, pread, mmap, pread_threadpool.", 0) \ M(String, remote_filesystem_read_method, "threadpool", "Method of reading data from remote filesystem, one of: read, threadpool.", 0) \ M(Bool, local_filesystem_read_prefetch, false, "Should use prefetching when reading data from local filesystem.", 0) \ M(Bool, remote_filesystem_read_prefetch, true, "Should use prefetching when reading data from remote filesystem.", 0) \ From aa8f282ad95d92b5e89838e228afbcb67cf2865c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Feb 2022 18:43:02 +0100 Subject: [PATCH 0002/1954] Switch to clang/llvm 14 --- docker/packager/binary/Dockerfile | 2 +- docker/packager/deb/Dockerfile | 2 +- docker/packager/packager | 6 ++-- docker/test/base/Dockerfile | 2 +- docker/test/codebrowser/Dockerfile | 2 +- docker/test/fasttest/Dockerfile | 2 +- docker/test/fuzzer/run-fuzzer.sh | 4 +-- docker/test/keeper-jepsen/run.sh | 2 +- docs/_includes/cmake_in_clickhouse_header.md | 4 +-- docs/en/development/build-cross-arm.md | 4 +-- docs/en/development/build-cross-osx.md | 6 ++-- docs/en/development/build-cross-riscv.md | 2 +- docs/en/development/build.md | 2 +- docs/en/development/developer-instruction.md | 2 +- docs/ru/development/developer-instruction.md | 8 ----- tests/ci/ci_config.json | 12 ++++---- tests/ci/ci_config.py | 32 ++++++++++---------- 17 files changed, 42 insertions(+), 52 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index e3e2e689b17..71dfca9fa4e 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -6,7 +6,7 @@ FROM ubuntu:20.04 ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14 RUN apt-get update \ && apt-get install \ diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 76a5f1d91c0..e150f37b6c9 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -6,7 +6,7 @@ FROM ubuntu:20.04 ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14 RUN apt-get update \ && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \ diff --git a/docker/packager/packager b/docker/packager/packager index 05b2e02df96..4c80411951e 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -196,10 +196,8 @@ if __name__ == "__main__": parser.add_argument("--clickhouse-repo-path", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) parser.add_argument("--output-dir", required=True) parser.add_argument("--build-type", choices=("debug", ""), default="") - parser.add_argument("--compiler", choices=("clang-11", "clang-11-darwin", "clang-11-darwin-aarch64", "clang-11-aarch64", - "clang-12", "clang-12-darwin", "clang-12-darwin-aarch64", "clang-12-aarch64", - "clang-13", "clang-13-darwin", "clang-13-darwin-aarch64", "clang-13-aarch64", "clang-13-ppc64le", - "clang-11-freebsd", "clang-12-freebsd", "clang-13-freebsd", "gcc-11"), default="clang-13") + parser.add_argument("--compiler", choices=("clang-14", "clang-14-darwin", "clang-14-darwin-aarch64", "clang-14-aarch64", "clang-14-ppc64le", + "clang-14-freebsd", "gcc-11"), default="clang-14") parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="") parser.add_argument("--split-binary", action="store_true") parser.add_argument("--clang-tidy", action="store_true") diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index 6beab2e5bb7..ca44354620f 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -7,7 +7,7 @@ FROM clickhouse/test-util:$FROM_TAG ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14 RUN apt-get update \ && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \ diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile index 97f3f54ad98..102c2d4c697 100644 --- a/docker/test/codebrowser/Dockerfile +++ b/docker/test/codebrowser/Dockerfile @@ -8,7 +8,7 @@ FROM clickhouse/binary-builder:$FROM_TAG ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list -RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-13 libllvm13 libclang-13-dev libmlir-13-dev +RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-14 libllvm14 libclang-14-dev libmlir-14-dev # repo versions doesn't work correctly with C++17 # also we push reports to s3, so we add index.html to subfolder urls diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 46b74d89e13..03a79b45a10 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -7,7 +7,7 @@ FROM clickhouse/test-util:$FROM_TAG ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14 RUN apt-get update \ && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \ diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index e18c07bf2c1..d8cb417f18b 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -12,8 +12,8 @@ stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "$script_dir" repo_dir=ch -BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-13_debug_none_bundled_unsplitted_disable_False_binary"} -BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"} +BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-14_debug_none_bundled_unsplitted_disable_False_binary"} +BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"} function clone { diff --git a/docker/test/keeper-jepsen/run.sh b/docker/test/keeper-jepsen/run.sh index d7534270e2c..53ef63f33b4 100644 --- a/docker/test/keeper-jepsen/run.sh +++ b/docker/test/keeper-jepsen/run.sh @@ -2,7 +2,7 @@ set -euo pipefail -CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-13_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"} +CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-14_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"} CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""} diff --git a/docs/_includes/cmake_in_clickhouse_header.md b/docs/_includes/cmake_in_clickhouse_header.md index 02019f13964..de4b1ef7af5 100644 --- a/docs/_includes/cmake_in_clickhouse_header.md +++ b/docs/_includes/cmake_in_clickhouse_header.md @@ -6,8 +6,8 @@ Minimal ClickHouse build example: ```bash cmake .. \ - -DCMAKE_C_COMPILER=$(which clang-13) \ - -DCMAKE_CXX_COMPILER=$(which clang++-13) \ + -DCMAKE_C_COMPILER=$(which clang-14) \ + -DCMAKE_CXX_COMPILER=$(which clang++-14) \ -DCMAKE_BUILD_TYPE=Debug \ -DENABLE_CLICKHOUSE_ALL=OFF \ -DENABLE_CLICKHOUSE_SERVER=ON \ diff --git a/docs/en/development/build-cross-arm.md b/docs/en/development/build-cross-arm.md index eb99105a857..bf378cb7712 100644 --- a/docs/en/development/build-cross-arm.md +++ b/docs/en/development/build-cross-arm.md @@ -9,7 +9,7 @@ This is for the case when you have Linux machine and want to use it to build `cl The cross-build for AARCH64 is based on the [Build instructions](../development/build.md), follow them first. -## Install Clang-13 +## Install Clang-14 or newer Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup or do ``` @@ -30,7 +30,7 @@ tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build-aarch64/cma ``` bash cd ClickHouse mkdir build-arm64 -CC=clang-13 CXX=clang++-13 cmake . -Bbuild-arm64 -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-aarch64.cmake +CC=clang-14 CXX=clang++-14 cmake . -Bbuild-arm64 -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-aarch64.cmake ninja -C build-arm64 ``` diff --git a/docs/en/development/build-cross-osx.md b/docs/en/development/build-cross-osx.md index c7e40013113..e786843552d 100644 --- a/docs/en/development/build-cross-osx.md +++ b/docs/en/development/build-cross-osx.md @@ -9,14 +9,14 @@ This is for the case when you have Linux machine and want to use it to build `cl The cross-build for Mac OS X is based on the [Build instructions](../development/build.md), follow them first. -## Install Clang-13 +## Install Clang-14 Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup. For example the commands for Bionic are like: ``` bash -sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-13 main" >> /etc/apt/sources.list -sudo apt-get install clang-13 +sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-14 main" >> /etc/apt/sources.list +sudo apt-get install clang-14 ``` ## Install Cross-Compilation Toolset {#install-cross-compilation-toolset} diff --git a/docs/en/development/build-cross-riscv.md b/docs/en/development/build-cross-riscv.md index 5cdce710b41..6fab85314f8 100644 --- a/docs/en/development/build-cross-riscv.md +++ b/docs/en/development/build-cross-riscv.md @@ -23,7 +23,7 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ``` bash cd ClickHouse mkdir build-riscv64 -CC=clang-13 CXX=clang++-13 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DENABLE_ORC=OFF -DUSE_UNWIND=OFF -DENABLE_GRPC=OFF -DENABLE_HDFS=OFF -DENABLE_MYSQL=OFF +CC=clang-14 CXX=clang++-14 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DENABLE_ORC=OFF -DUSE_UNWIND=OFF -DENABLE_GRPC=OFF -DENABLE_HDFS=OFF -DENABLE_MYSQL=OFF ninja -C build-riscv64 ``` diff --git a/docs/en/development/build.md b/docs/en/development/build.md index aaa3bdfd043..19401ba3766 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -76,7 +76,7 @@ The build requires the following components: - Git (is used only to checkout the sources, it’s not needed for the build) - CMake 3.10 or newer - Ninja -- C++ compiler: clang-13 or newer +- C++ compiler: clang-14 or newer - Linker: lld If all the components are installed, you may build in the same way as the steps above. diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index f7d7100d181..a23617365a5 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -154,7 +154,7 @@ While inside the `build` directory, configure your build by running CMake. Befor export CC=clang CXX=clang++ cmake .. -If you installed clang using the automatic installation script above, also specify the version of clang installed in the first command, e.g. `export CC=clang-13 CXX=clang++-13`. The clang version will be in the script output. +If you installed clang using the automatic installation script above, also specify the version of clang installed in the first command, e.g. `export CC=clang-14 CXX=clang++-14`. The clang version will be in the script output. The `CC` variable specifies the compiler for C (short for C Compiler), and `CXX` variable instructs which C++ compiler is to be used for building. diff --git a/docs/ru/development/developer-instruction.md b/docs/ru/development/developer-instruction.md index 964d39163d8..92e80578c49 100644 --- a/docs/ru/development/developer-instruction.md +++ b/docs/ru/development/developer-instruction.md @@ -200,14 +200,6 @@ cmake -DUSE_DEBUG_HELPERS=1 -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 В процессе сборки могут появится сообщения `libprotobuf WARNING` про protobuf файлы в библиотеке libhdfs2. Это не имеет значения. -В случае получения ошибок вида `error: variable 'y' set but not used [-Werror,-Wunused-but-set-variable]` ножно попробовать использовать другую версию компилятора сlang. Например, на момент написания данного текста описанная выше команда по установке clang для Ubuntu 20.04 по-умолчанию устанавливает clang-13, с которым возникает эта ошибка. Для решения проблемы можно установить clang-12 с помощью команд: -```bash -wget https://apt.llvm.org/llvm.sh -chmod +x llvm.sh -sudo ./llvm.sh 12 -``` -И далее использовать именно его, указав соответствующую версию при установке переменных окружения CC и CXX перед вызовом cmake. - При успешной сборке, вы получите готовый исполняемый файл `ClickHouse/build/programs/clickhouse`: ls -l programs/clickhouse diff --git a/tests/ci/ci_config.json b/tests/ci/ci_config.json index 19afdd172d5..81ed2fff3d2 100644 --- a/tests/ci/ci_config.json +++ b/tests/ci/ci_config.json @@ -1,7 +1,7 @@ { "build_config": [ { - "compiler": "clang-13", + "compiler": "clang-14", "build-type": "", "sanitizer": "", "package-type": "deb", @@ -12,7 +12,7 @@ "with_coverage": false }, { - "compiler": "clang-13", + "compiler": "clang-14", "build-type": "", "sanitizer": "", "package-type": "performance", @@ -32,7 +32,7 @@ "with_coverage": false }, { - "compiler": "clang-13", + "compiler": "clang-14", "build-type": "", "sanitizer": "", "package-type": "binary", @@ -45,7 +45,7 @@ "tests_config": { "Testflows check": { "required_build_properties": { - "compiler": "clang-13", + "compiler": "clang-14", "package_type": "deb", "build_type": "relwithdebuginfo", "sanitizer": "none", @@ -57,7 +57,7 @@ }, "Release": { "required_build_properties": { - "compiler": "clang-13", + "compiler": "clang-14", "package_type": "deb", "build_type": "relwithdebuginfo", "sanitizer": "none", @@ -69,7 +69,7 @@ }, "ClickHouse Keeper Jepsen": { "required_build_properties": { - "compiler": "clang-13", + "compiler": "clang-14", "package_type": "binary", "build_type": "relwithdebuginfo", "sanitizer": "none", diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 000d3d9a000..9afb5e981f6 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -8,7 +8,7 @@ BuildConfig = Dict[str, ConfValue] CI_CONFIG = { "build_config": { "package_release": { - "compiler": "clang-13", + "compiler": "clang-14", "build_type": "", "sanitizer": "", "package_type": "deb", @@ -19,7 +19,7 @@ CI_CONFIG = { "with_coverage": False, }, "performance": { - "compiler": "clang-13", + "compiler": "clang-14", "build_type": "", "sanitizer": "", "package_type": "performance", @@ -39,7 +39,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_aarch64": { - "compiler": "clang-13-aarch64", + "compiler": "clang-14-aarch64", "build_type": "", "sanitizer": "", "package_type": "deb", @@ -50,7 +50,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_asan": { - "compiler": "clang-13", + "compiler": "clang-14", "build_type": "", "sanitizer": "address", "package_type": "deb", @@ -60,7 +60,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_ubsan": { - "compiler": "clang-13", + "compiler": "clang-14", "build_type": "", "sanitizer": "undefined", "package_type": "deb", @@ -70,7 +70,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_tsan": { - "compiler": "clang-13", + "compiler": "clang-14", "build_type": "", "sanitizer": "thread", "package_type": "deb", @@ -80,7 +80,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_msan": { - "compiler": "clang-13", + "compiler": "clang-14", "build_type": "", "sanitizer": "memory", "package_type": "deb", @@ -90,7 +90,7 @@ CI_CONFIG = { "with_coverage": False, }, "package_debug": { - "compiler": "clang-13", + "compiler": "clang-14", "build_type": "debug", "sanitizer": "", "package_type": "deb", @@ -100,7 +100,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_release": { - "compiler": "clang-13", + "compiler": "clang-14", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -111,7 +111,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_tidy": { - "compiler": "clang-13", + "compiler": "clang-14", "build_type": "debug", "sanitizer": "", "package_type": "binary", @@ -122,7 +122,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_splitted": { - "compiler": "clang-13", + "compiler": "clang-14", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -132,7 +132,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_darwin": { - "compiler": "clang-13-darwin", + "compiler": "clang-14-darwin", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -143,7 +143,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_aarch64": { - "compiler": "clang-13-aarch64", + "compiler": "clang-14-aarch64", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -154,7 +154,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_freebsd": { - "compiler": "clang-13-freebsd", + "compiler": "clang-14-freebsd", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -165,7 +165,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_darwin_aarch64": { - "compiler": "clang-13-darwin-aarch64", + "compiler": "clang-14-darwin-aarch64", "build_type": "", "sanitizer": "", "package_type": "binary", @@ -176,7 +176,7 @@ CI_CONFIG = { "with_coverage": False, }, "binary_ppc64le": { - "compiler": "clang-13-ppc64le", + "compiler": "clang-14-ppc64le", "build_type": "", "sanitizer": "", "package_type": "binary", From 7671d2f1d5598cdf8685d19166fb7013279168d7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 12 Mar 2022 00:45:26 +0100 Subject: [PATCH 0003/1954] Change timezone in stateless tests --- ...s_and_insert_without_explicit_database.sql | 4 +- .../0_stateless/00189_time_zones_long.sql | 144 ++++++------ .../00206_empty_array_to_single.sql | 2 +- tests/queries/0_stateless/00301_csv.sh | 4 +- ...4_json_each_row_input_with_noisy_fields.sh | 2 +- tests/queries/0_stateless/00502_sum_map.sql | 2 +- .../00506_shard_global_in_union.sql | 2 +- .../00512_fractional_time_zones.sh | 2 +- .../0_stateless/00515_enhanced_time_zones.sql | 18 +- tests/queries/0_stateless/00538_datediff.sql | 24 +- .../0_stateless/00561_storage_join.sql | 2 +- .../0_stateless/00718_format_datetime.sql | 2 +- .../00719_format_datetime_rand.sql | 2 +- .../00735_long_conditional.reference | 12 +- .../0_stateless/00735_long_conditional.sql | 96 ++++---- ...01_daylight_saving_time_hour_underflow.sql | 2 +- ...aving_time_shift_backwards_at_midnight.sql | 2 +- .../00825_protobuf_format_persons.sh | 6 +- .../0_stateless/00835_if_generic_case.sql | 24 +- .../00910_decimal_group_array_crash_3783.sql | 14 +- ...00921_datetime64_compatibility_long.python | 130 +++++------ ...21_datetime64_compatibility_long.reference | 212 +++++++++--------- .../00927_asof_join_other_types.sh | 2 +- .../00935_to_iso_week_first_year.sql | 2 +- .../0_stateless/00941_to_custom_week.sql | 10 +- .../0_stateless/00945_bloom_filter_index.sql | 46 ++-- .../01077_mutations_index_consistency.sh | 12 +- .../0_stateless/01087_storage_generate.sql | 2 +- .../01087_table_function_generate.reference | 4 +- .../01087_table_function_generate.sql | 16 +- .../0_stateless/01098_msgpack_format.sh | 2 +- .../01186_conversion_to_nullable.sql | 6 +- ...ter_rename_with_default_zookeeper_long.sql | 4 +- .../0_stateless/01269_toStartOfSecond.sql | 2 +- tests/queries/0_stateless/01273_arrow_load.sh | 2 +- .../0_stateless/01277_toUnixTimestamp64.sql | 12 +- .../0_stateless/01280_min_map_max_map.sql | 4 +- .../0_stateless/01307_orc_output_format.sh | 2 +- .../01379_with_fill_several_columns.sql | 8 +- ...396_negative_datetime_saturate_to_zero.sql | 2 +- .../01414_low_cardinality_nullable.sql | 30 +-- ..._parse_date_time_best_effort_timestamp.sql | 6 +- .../01440_to_date_monotonicity.sql | 8 +- .../01442_date_time_with_params.reference | 6 +- .../01442_date_time_with_params.sql | 6 +- .../01508_partition_pruning_long.queries | 38 ++-- .../01508_partition_pruning_long.reference | 4 +- .../01516_date_time_output_format.sql | 8 +- .../0_stateless/01582_any_join_supertype.sql | 2 +- .../01615_two_args_function_index_fix.sql | 2 +- .../0_stateless/01676_reinterpret_as.sql | 4 +- .../01691_DateTime64_clamp.reference | 26 +-- .../0_stateless/01691_DateTime64_clamp.sql | 26 +-- .../01692_DateTime64_from_DateTime.sql | 6 +- .../0_stateless/01698_fix_toMinute.reference | 2 +- .../0_stateless/01698_fix_toMinute.sql | 6 +- .../01699_timezoneOffset.reference | 4 +- .../0_stateless/01699_timezoneOffset.sql | 16 +- .../01702_toDateTime_from_string_clamping.sql | 8 +- ...732_more_consistent_datetime64_parsing.sql | 8 +- .../01734_datetime64_from_float.sql | 6 +- .../0_stateless/01761_round_year_bounds.sql | 2 +- .../0_stateless/01769_extended_range_2.sql | 2 +- .../01772_to_start_of_hour_align.sql | 4 +- ...ormatDateTime_DateTime64_century.reference | 24 +- ...1802_formatDateTime_DateTime64_century.sql | 24 +- .../01802_toDateTime64_large_values.reference | 6 +- .../01802_toDateTime64_large_values.sql | 6 +- tests/queries/0_stateless/01811_datename.sql | 8 +- .../0_stateless/01821_to_date_time_ubsan.sql | 4 +- .../0_stateless/01852_map_combinator.sql | 2 +- ...1867_support_datetime64_version_column.sql | 2 +- .../01868_order_by_fill_with_datetime64.sql | 4 +- .../0_stateless/01891_partition_hash.sql | 2 +- .../01891_partition_hash_no_long_int.sql | 2 +- .../0_stateless/01905_to_json_string.sql | 2 +- .../0_stateless/01921_datatype_date32.sql | 18 +- .../01925_date_date_time_comparison.sql | 4 +- .../01926_date_date_time_supertype.reference | 14 +- .../01926_date_date_time_supertype.sql | 10 +- ...nversion_between_date32_and_datetime64.sql | 2 +- .../02096_date_time_1970_saturation.sql | 24 +- .../02176_toStartOfWeek_overflow_pruning.sql | 2 +- .../02184_default_table_engine.sql | 4 +- .../alltypes_list.parquet.columns | 2 +- ...1.column-metadata-handling.parquet.columns | 2 +- .../00900_parquet_create_table_columns.py | 4 +- 87 files changed, 624 insertions(+), 624 deletions(-) diff --git a/tests/queries/0_stateless/00101_materialized_views_and_insert_without_explicit_database.sql b/tests/queries/0_stateless/00101_materialized_views_and_insert_without_explicit_database.sql index 2fd097b9538..f3130f24521 100644 --- a/tests/queries/0_stateless/00101_materialized_views_and_insert_without_explicit_database.sql +++ b/tests/queries/0_stateless/00101_materialized_views_and_insert_without_explicit_database.sql @@ -8,8 +8,8 @@ DROP TABLE IF EXISTS test_table; DROP TABLE IF EXISTS test_view; DROP TABLE IF EXISTS test_view_filtered; -CREATE TABLE test_table (EventDate Date, CounterID UInt32, UserID UInt64, EventTime DateTime('Europe/Moscow'), UTCEventTime DateTime('UTC')) ENGINE = MergeTree(EventDate, CounterID, 8192); -CREATE MATERIALIZED VIEW test_view (Rows UInt64, MaxHitTime DateTime('Europe/Moscow')) ENGINE = Memory AS SELECT count() AS Rows, max(UTCEventTime) AS MaxHitTime FROM test_table; +CREATE TABLE test_table (EventDate Date, CounterID UInt32, UserID UInt64, EventTime DateTime('Asia/Istanbul'), UTCEventTime DateTime('UTC')) ENGINE = MergeTree(EventDate, CounterID, 8192); +CREATE MATERIALIZED VIEW test_view (Rows UInt64, MaxHitTime DateTime('Asia/Istanbul')) ENGINE = Memory AS SELECT count() AS Rows, max(UTCEventTime) AS MaxHitTime FROM test_table; CREATE MATERIALIZED VIEW test_view_filtered (EventDate Date, CounterID UInt32) ENGINE = Memory POPULATE AS SELECT CounterID, EventDate FROM test_table WHERE EventDate < '2013-01-01'; INSERT INTO test_table (EventDate, UTCEventTime) VALUES ('2014-01-02', '2014-01-02 03:04:06'); diff --git a/tests/queries/0_stateless/00189_time_zones_long.sql b/tests/queries/0_stateless/00189_time_zones_long.sql index ecc5f62ed1d..a25c9c7a415 100644 --- a/tests/queries/0_stateless/00189_time_zones_long.sql +++ b/tests/queries/0_stateless/00189_time_zones_long.sql @@ -1,12 +1,12 @@ -- Tags: long -/* timestamp 1419800400 == 2014-12-29 00:00:00 (Europe/Moscow) */ -/* timestamp 1412106600 == 2014-09-30 23:50:00 (Europe/Moscow) */ -/* timestamp 1420102800 == 2015-01-01 12:00:00 (Europe/Moscow) */ -/* timestamp 1428310800 == 2015-04-06 12:00:00 (Europe/Moscow) */ -/* timestamp 1436956200 == 2015-07-15 13:30:00 (Europe/Moscow) */ -/* timestamp 1426415400 == 2015-03-15 13:30:00 (Europe/Moscow) */ -/* timestamp 1549483055 == 2019-02-06 22:57:35 (Europe/Moscow) */ +/* timestamp 1419800400 == 2014-12-29 00:00:00 (Asia/Istanbul) */ +/* timestamp 1412106600 == 2014-09-30 23:50:00 (Asia/Istanbul) */ +/* timestamp 1420102800 == 2015-01-01 12:00:00 (Asia/Istanbul) */ +/* timestamp 1428310800 == 2015-04-06 12:00:00 (Asia/Istanbul) */ +/* timestamp 1436956200 == 2015-07-15 13:30:00 (Asia/Istanbul) */ +/* timestamp 1426415400 == 2015-03-15 13:30:00 (Asia/Istanbul) */ +/* timestamp 1549483055 == 2019-02-06 22:57:35 (Asia/Istanbul) */ /* date 16343 == 2014-09-30 */ /* date 16433 == 2014-12-29 */ /* date 17933 == 2019-02-06 */ @@ -14,12 +14,12 @@ /* toStartOfDay */ SELECT 'toStartOfDay'; -SELECT toStartOfDay(toDateTime(1412106600), 'Europe/Moscow'); +SELECT toStartOfDay(toDateTime(1412106600), 'Asia/Istanbul'); SELECT toStartOfDay(toDateTime(1412106600), 'Europe/Paris'); SELECT toStartOfDay(toDateTime(1412106600), 'Europe/London'); SELECT toStartOfDay(toDateTime(1412106600), 'Asia/Tokyo'); SELECT toStartOfDay(toDateTime(1412106600), 'Pacific/Pitcairn'); -SELECT toStartOfDay(toDate(16343), 'Europe/Moscow'); +SELECT toStartOfDay(toDate(16343), 'Asia/Istanbul'); SELECT toStartOfDay(toDate(16343), 'Europe/Paris'); SELECT toStartOfDay(toDate(16343), 'Europe/London'); SELECT toStartOfDay(toDate(16343), 'Asia/Tokyo'); @@ -28,7 +28,7 @@ SELECT toStartOfDay(toDate(16343), 'Pacific/Pitcairn'); /* toMonday */ SELECT 'toMonday'; -SELECT toMonday(toDateTime(1419800400), 'Europe/Moscow'); +SELECT toMonday(toDateTime(1419800400), 'Asia/Istanbul'); SELECT toMonday(toDateTime(1419800400), 'Europe/Paris'); SELECT toMonday(toDateTime(1419800400), 'Europe/London'); SELECT toMonday(toDateTime(1419800400), 'Asia/Tokyo'); @@ -42,7 +42,7 @@ SELECT toMonday(toDate(16433)); /* toStartOfMonth */ SELECT 'toStartOfMonth'; -SELECT toStartOfMonth(toDateTime(1419800400), 'Europe/Moscow'); +SELECT toStartOfMonth(toDateTime(1419800400), 'Asia/Istanbul'); SELECT toStartOfMonth(toDateTime(1419800400), 'Europe/Paris'); SELECT toStartOfMonth(toDateTime(1419800400), 'Europe/London'); SELECT toStartOfMonth(toDateTime(1419800400), 'Asia/Tokyo'); @@ -56,7 +56,7 @@ SELECT toStartOfMonth(toDate(16433)); /* toStartOfQuarter */ SELECT 'toStartOfQuarter'; -SELECT toStartOfQuarter(toDateTime(1412106600), 'Europe/Moscow'); +SELECT toStartOfQuarter(toDateTime(1412106600), 'Asia/Istanbul'); SELECT toStartOfQuarter(toDateTime(1412106600), 'Europe/Paris'); SELECT toStartOfQuarter(toDateTime(1412106600), 'Europe/London'); SELECT toStartOfQuarter(toDateTime(1412106600), 'Asia/Tokyo'); @@ -70,7 +70,7 @@ SELECT toStartOfQuarter(toDate(16343)); /* toStartOfYear */ SELECT 'toStartOfYear'; -SELECT toStartOfYear(toDateTime(1419800400), 'Europe/Moscow'); +SELECT toStartOfYear(toDateTime(1419800400), 'Asia/Istanbul'); SELECT toStartOfYear(toDateTime(1419800400), 'Europe/Paris'); SELECT toStartOfYear(toDateTime(1419800400), 'Europe/London'); SELECT toStartOfYear(toDateTime(1419800400), 'Asia/Tokyo'); @@ -84,7 +84,7 @@ SELECT toStartOfYear(toDate(16433)); /* toTime */ SELECT 'toTime'; -SELECT toString(toTime(toDateTime(1420102800), 'Europe/Moscow'), 'Europe/Moscow'), toString(toTime(toDateTime(1428310800), 'Europe/Moscow'), 'Europe/Moscow'); +SELECT toString(toTime(toDateTime(1420102800), 'Asia/Istanbul'), 'Asia/Istanbul'), toString(toTime(toDateTime(1428310800), 'Asia/Istanbul'), 'Asia/Istanbul'); SELECT toString(toTime(toDateTime(1420102800), 'Europe/Paris'), 'Europe/Paris'), toString(toTime(toDateTime(1428310800), 'Europe/Paris'), 'Europe/Paris'); SELECT toString(toTime(toDateTime(1420102800), 'Europe/London'), 'Europe/London'), toString(toTime(toDateTime(1428310800), 'Europe/London'), 'Europe/London'); SELECT toString(toTime(toDateTime(1420102800), 'Asia/Tokyo'), 'Asia/Tokyo'), toString(toTime(toDateTime(1428310800), 'Asia/Tokyo'), 'Asia/Tokyo'); @@ -93,7 +93,7 @@ SELECT toString(toTime(toDateTime(1420102800), 'Pacific/Pitcairn'), 'Pacific/Pit /* toYear */ SELECT 'toYear'; -SELECT toYear(toDateTime(1412106600), 'Europe/Moscow'); +SELECT toYear(toDateTime(1412106600), 'Asia/Istanbul'); SELECT toYear(toDateTime(1412106600), 'Europe/Paris'); SELECT toYear(toDateTime(1412106600), 'Europe/London'); SELECT toYear(toDateTime(1412106600), 'Asia/Tokyo'); @@ -102,7 +102,7 @@ SELECT toYear(toDateTime(1412106600), 'Pacific/Pitcairn'); /* toMonth */ SELECT 'toMonth'; -SELECT toMonth(toDateTime(1412106600), 'Europe/Moscow'); +SELECT toMonth(toDateTime(1412106600), 'Asia/Istanbul'); SELECT toMonth(toDateTime(1412106600), 'Europe/Paris'); SELECT toMonth(toDateTime(1412106600), 'Europe/London'); SELECT toMonth(toDateTime(1412106600), 'Asia/Tokyo'); @@ -111,7 +111,7 @@ SELECT toMonth(toDateTime(1412106600), 'Pacific/Pitcairn'); /* toDayOfMonth */ SELECT 'toDayOfMonth'; -SELECT toDayOfMonth(toDateTime(1412106600), 'Europe/Moscow'); +SELECT toDayOfMonth(toDateTime(1412106600), 'Asia/Istanbul'); SELECT toDayOfMonth(toDateTime(1412106600), 'Europe/Paris'); SELECT toDayOfMonth(toDateTime(1412106600), 'Europe/London'); SELECT toDayOfMonth(toDateTime(1412106600), 'Asia/Tokyo'); @@ -120,7 +120,7 @@ SELECT toDayOfMonth(toDateTime(1412106600), 'Pacific/Pitcairn'); /* toDayOfWeek */ SELECT 'toDayOfWeek'; -SELECT toDayOfWeek(toDateTime(1412106600), 'Europe/Moscow'); +SELECT toDayOfWeek(toDateTime(1412106600), 'Asia/Istanbul'); SELECT toDayOfWeek(toDateTime(1412106600), 'Europe/Paris'); SELECT toDayOfWeek(toDateTime(1412106600), 'Europe/London'); SELECT toDayOfWeek(toDateTime(1412106600), 'Asia/Tokyo'); @@ -129,7 +129,7 @@ SELECT toDayOfWeek(toDateTime(1412106600), 'Pacific/Pitcairn'); /* toHour */ SELECT 'toHour'; -SELECT toHour(toDateTime(1412106600), 'Europe/Moscow'); +SELECT toHour(toDateTime(1412106600), 'Asia/Istanbul'); SELECT toHour(toDateTime(1412106600), 'Europe/Paris'); SELECT toHour(toDateTime(1412106600), 'Europe/London'); SELECT toHour(toDateTime(1412106600), 'Asia/Tokyo'); @@ -138,7 +138,7 @@ SELECT toHour(toDateTime(1412106600), 'Pacific/Pitcairn'); /* toMinute */ SELECT 'toMinute'; -SELECT toMinute(toDateTime(1412106600), 'Europe/Moscow'); +SELECT toMinute(toDateTime(1412106600), 'Asia/Istanbul'); SELECT toMinute(toDateTime(1412106600), 'Europe/Paris'); SELECT toMinute(toDateTime(1412106600), 'Europe/London'); SELECT toMinute(toDateTime(1412106600), 'Asia/Tokyo'); @@ -147,7 +147,7 @@ SELECT toMinute(toDateTime(1412106600), 'Pacific/Pitcairn'); /* toSecond */ SELECT 'toSecond'; -SELECT toSecond(toDateTime(1412106600), 'Europe/Moscow'); +SELECT toSecond(toDateTime(1412106600), 'Asia/Istanbul'); SELECT toSecond(toDateTime(1412106600), 'Europe/Paris'); SELECT toSecond(toDateTime(1412106600), 'Europe/London'); SELECT toSecond(toDateTime(1412106600), 'Asia/Tokyo'); @@ -156,7 +156,7 @@ SELECT toSecond(toDateTime(1412106600), 'Pacific/Pitcairn'); /* toStartOfMinute */ SELECT 'toStartOfMinute'; -SELECT toString(toStartOfMinute(toDateTime(1549483055), 'Europe/Moscow'), 'Europe/Moscow'); +SELECT toString(toStartOfMinute(toDateTime(1549483055), 'Asia/Istanbul'), 'Asia/Istanbul'); SELECT toString(toStartOfMinute(toDateTime(1549483055), 'Europe/Paris'), 'Europe/Paris'); SELECT toString(toStartOfMinute(toDateTime(1549483055), 'Europe/London'), 'Europe/London'); SELECT toString(toStartOfMinute(toDateTime(1549483055), 'Asia/Tokyo'), 'Asia/Tokyo'); @@ -165,7 +165,7 @@ SELECT toString(toStartOfMinute(toDateTime(1549483055), 'Pacific/Pitcairn'), 'Pa /* toStartOfFiveMinute */ SELECT 'toStartOfFiveMinute'; -SELECT toString(toStartOfFiveMinute(toDateTime(1549483055), 'Europe/Moscow'), 'Europe/Moscow'); +SELECT toString(toStartOfFiveMinute(toDateTime(1549483055), 'Asia/Istanbul'), 'Asia/Istanbul'); SELECT toString(toStartOfFiveMinute(toDateTime(1549483055), 'Europe/Paris'), 'Europe/Paris'); SELECT toString(toStartOfFiveMinute(toDateTime(1549483055), 'Europe/London'), 'Europe/London'); SELECT toString(toStartOfFiveMinute(toDateTime(1549483055), 'Asia/Tokyo'), 'Asia/Tokyo'); @@ -174,7 +174,7 @@ SELECT toString(toStartOfFiveMinute(toDateTime(1549483055), 'Pacific/Pitcairn'), /* toStartOfTenMinutes */ SELECT 'toStartOfTenMinutes'; -SELECT toString(toStartOfTenMinutes(toDateTime(1549483055), 'Europe/Moscow'), 'Europe/Moscow'); +SELECT toString(toStartOfTenMinutes(toDateTime(1549483055), 'Asia/Istanbul'), 'Asia/Istanbul'); SELECT toString(toStartOfTenMinutes(toDateTime(1549483055), 'Europe/Paris'), 'Europe/Paris'); SELECT toString(toStartOfTenMinutes(toDateTime(1549483055), 'Europe/London'), 'Europe/London'); SELECT toString(toStartOfTenMinutes(toDateTime(1549483055), 'Asia/Tokyo'), 'Asia/Tokyo'); @@ -183,7 +183,7 @@ SELECT toString(toStartOfTenMinutes(toDateTime(1549483055), 'Pacific/Pitcairn'), /* toStartOfFifteenMinutes */ SELECT 'toStartOfFifteenMinutes'; -SELECT toString(toStartOfFifteenMinutes(toDateTime(1549483055), 'Europe/Moscow'), 'Europe/Moscow'); +SELECT toString(toStartOfFifteenMinutes(toDateTime(1549483055), 'Asia/Istanbul'), 'Asia/Istanbul'); SELECT toString(toStartOfFifteenMinutes(toDateTime(1549483055), 'Europe/Paris'), 'Europe/Paris'); SELECT toString(toStartOfFifteenMinutes(toDateTime(1549483055), 'Europe/London'), 'Europe/London'); SELECT toString(toStartOfFifteenMinutes(toDateTime(1549483055), 'Asia/Tokyo'), 'Asia/Tokyo'); @@ -192,7 +192,7 @@ SELECT toString(toStartOfFifteenMinutes(toDateTime(1549483055), 'Pacific/Pitcair /* toStartOfHour */ SELECT 'toStartOfHour'; -SELECT toString(toStartOfHour(toDateTime(1549483055), 'Europe/Moscow'), 'Europe/Moscow'); +SELECT toString(toStartOfHour(toDateTime(1549483055), 'Asia/Istanbul'), 'Asia/Istanbul'); SELECT toString(toStartOfHour(toDateTime(1549483055), 'Europe/Paris'), 'Europe/Paris'); SELECT toString(toStartOfHour(toDateTime(1549483055), 'Europe/London'), 'Europe/London'); SELECT toString(toStartOfHour(toDateTime(1549483055), 'Asia/Tokyo'), 'Asia/Tokyo'); @@ -201,33 +201,33 @@ SELECT toString(toStartOfHour(toDateTime(1549483055), 'Pacific/Pitcairn'), 'Paci /* toStartOfInterval */ SELECT 'toStartOfInterval'; -SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 1 year, 'Europe/Moscow'); -SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 2 year, 'Europe/Moscow'); -SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 5 year, 'Europe/Moscow'); -SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 1 quarter, 'Europe/Moscow'); -SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 2 quarter, 'Europe/Moscow'); -SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 3 quarter, 'Europe/Moscow'); -SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 1 month, 'Europe/Moscow'); -SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 2 month, 'Europe/Moscow'); -SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 5 month, 'Europe/Moscow'); -SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 1 week, 'Europe/Moscow'); -SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 2 week, 'Europe/Moscow'); -SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 6 week, 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 1 day, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 2 day, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 5 day, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 1 hour, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 2 hour, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 6 hour, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 24 hour, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 1 minute, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 2 minute, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 5 minute, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 20 minute, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 90 minute, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 1 second, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 2 second, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 5 second, 'Europe/Moscow'), 'Europe/Moscow'); +SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 1 year, 'Asia/Istanbul'); +SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 2 year, 'Asia/Istanbul'); +SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 5 year, 'Asia/Istanbul'); +SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 1 quarter, 'Asia/Istanbul'); +SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 2 quarter, 'Asia/Istanbul'); +SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 3 quarter, 'Asia/Istanbul'); +SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 1 month, 'Asia/Istanbul'); +SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 2 month, 'Asia/Istanbul'); +SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 5 month, 'Asia/Istanbul'); +SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 1 week, 'Asia/Istanbul'); +SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 2 week, 'Asia/Istanbul'); +SELECT toStartOfInterval(toDateTime(1549483055), INTERVAL 6 week, 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 1 day, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 2 day, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 5 day, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 1 hour, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 2 hour, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 6 hour, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 24 hour, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 1 minute, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 2 minute, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 5 minute, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 20 minute, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 90 minute, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 1 second, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 2 second, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDateTime(1549483055), INTERVAL 5 second, 'Asia/Istanbul'), 'Asia/Istanbul'); SELECT toStartOfInterval(toDate(17933), INTERVAL 1 year); SELECT toStartOfInterval(toDate(17933), INTERVAL 2 year); SELECT toStartOfInterval(toDate(17933), INTERVAL 5 year); @@ -240,14 +240,14 @@ SELECT toStartOfInterval(toDate(17933), INTERVAL 5 month); SELECT toStartOfInterval(toDate(17933), INTERVAL 1 week); SELECT toStartOfInterval(toDate(17933), INTERVAL 2 week); SELECT toStartOfInterval(toDate(17933), INTERVAL 6 week); -SELECT toString(toStartOfInterval(toDate(17933), INTERVAL 1 day, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDate(17933), INTERVAL 2 day, 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toString(toStartOfInterval(toDate(17933), INTERVAL 5 day, 'Europe/Moscow'), 'Europe/Moscow'); +SELECT toString(toStartOfInterval(toDate(17933), INTERVAL 1 day, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDate(17933), INTERVAL 2 day, 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toString(toStartOfInterval(toDate(17933), INTERVAL 5 day, 'Asia/Istanbul'), 'Asia/Istanbul'); /* toRelativeYearNum */ SELECT 'toRelativeYearNum'; -SELECT toRelativeYearNum(toDateTime(1412106600), 'Europe/Moscow') - toRelativeYearNum(toDateTime(0), 'Europe/Moscow'); +SELECT toRelativeYearNum(toDateTime(1412106600), 'Asia/Istanbul') - toRelativeYearNum(toDateTime(0), 'Asia/Istanbul'); SELECT toRelativeYearNum(toDateTime(1412106600), 'Europe/Paris') - toRelativeYearNum(toDateTime(0), 'Europe/Paris'); SELECT toRelativeYearNum(toDateTime(1412106600), 'Europe/London') - toRelativeYearNum(toDateTime(0), 'Europe/London'); SELECT toRelativeYearNum(toDateTime(1412106600), 'Asia/Tokyo') - toRelativeYearNum(toDateTime(0), 'Asia/Tokyo'); @@ -256,7 +256,7 @@ SELECT toRelativeYearNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelativ /* toRelativeMonthNum */ SELECT 'toRelativeMonthNum'; -SELECT toRelativeMonthNum(toDateTime(1412106600), 'Europe/Moscow') - toRelativeMonthNum(toDateTime(0), 'Europe/Moscow'); +SELECT toRelativeMonthNum(toDateTime(1412106600), 'Asia/Istanbul') - toRelativeMonthNum(toDateTime(0), 'Asia/Istanbul'); SELECT toRelativeMonthNum(toDateTime(1412106600), 'Europe/Paris') - toRelativeMonthNum(toDateTime(0), 'Europe/Paris'); SELECT toRelativeMonthNum(toDateTime(1412106600), 'Europe/London') - toRelativeMonthNum(toDateTime(0), 'Europe/London'); SELECT toRelativeMonthNum(toDateTime(1412106600), 'Asia/Tokyo') - toRelativeMonthNum(toDateTime(0), 'Asia/Tokyo'); @@ -265,7 +265,7 @@ SELECT toRelativeMonthNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelati /* toRelativeWeekNum */ SELECT 'toRelativeWeekNum'; -SELECT toRelativeWeekNum(toDateTime(1412106600), 'Europe/Moscow') - toRelativeWeekNum(toDateTime(0), 'Europe/Moscow'); +SELECT toRelativeWeekNum(toDateTime(1412106600), 'Asia/Istanbul') - toRelativeWeekNum(toDateTime(0), 'Asia/Istanbul'); SELECT toRelativeWeekNum(toDateTime(1412106600), 'Europe/Paris') - toRelativeWeekNum(toDateTime(0), 'Europe/Paris'); SELECT toRelativeWeekNum(toDateTime(1412106600), 'Europe/London') - toRelativeWeekNum(toDateTime(0), 'Europe/London'); SELECT toRelativeWeekNum(toDateTime(1412106600), 'Asia/Tokyo') - toRelativeWeekNum(toDateTime(0), 'Asia/Tokyo'); @@ -274,7 +274,7 @@ SELECT toRelativeWeekNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelativ /* toRelativeDayNum */ SELECT 'toRelativeDayNum'; -SELECT toRelativeDayNum(toDateTime(1412106600), 'Europe/Moscow') - toRelativeDayNum(toDateTime(0), 'Europe/Moscow'); +SELECT toRelativeDayNum(toDateTime(1412106600), 'Asia/Istanbul') - toRelativeDayNum(toDateTime(0), 'Asia/Istanbul'); SELECT toRelativeDayNum(toDateTime(1412106600), 'Europe/Paris') - toRelativeDayNum(toDateTime(0), 'Europe/Paris'); SELECT toRelativeDayNum(toDateTime(1412106600), 'Europe/London') - toRelativeDayNum(toDateTime(0), 'Europe/London'); SELECT toRelativeDayNum(toDateTime(1412106600), 'Asia/Tokyo') - toRelativeDayNum(toDateTime(0), 'Asia/Tokyo'); @@ -284,7 +284,7 @@ SELECT toUInt16(toRelativeDayNum(toDateTime(1412106600), 'Pacific/Pitcairn') - t /* toRelativeHourNum */ SELECT 'toRelativeHourNum'; -SELECT toRelativeHourNum(toDateTime(1412106600), 'Europe/Moscow') - toRelativeHourNum(toDateTime(0), 'Europe/Moscow'); +SELECT toRelativeHourNum(toDateTime(1412106600), 'Asia/Istanbul') - toRelativeHourNum(toDateTime(0), 'Asia/Istanbul'); SELECT toRelativeHourNum(toDateTime(1412106600), 'Europe/Paris') - toRelativeHourNum(toDateTime(0), 'Europe/Paris'); SELECT toRelativeHourNum(toDateTime(1412106600), 'Europe/London') - toRelativeHourNum(toDateTime(0), 'Europe/London'); SELECT toRelativeHourNum(toDateTime(1412106600), 'Asia/Tokyo') - toRelativeHourNum(toDateTime(0), 'Asia/Tokyo'); @@ -293,7 +293,7 @@ SELECT toRelativeHourNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelativ /* toRelativeMinuteNum */ SELECT 'toRelativeMinuteNum'; -SELECT toRelativeMinuteNum(toDateTime(1412106600), 'Europe/Moscow') - toRelativeMinuteNum(toDateTime(0), 'Europe/Moscow'); +SELECT toRelativeMinuteNum(toDateTime(1412106600), 'Asia/Istanbul') - toRelativeMinuteNum(toDateTime(0), 'Asia/Istanbul'); SELECT toRelativeMinuteNum(toDateTime(1412106600), 'Europe/Paris') - toRelativeMinuteNum(toDateTime(0), 'Europe/Paris'); SELECT toRelativeMinuteNum(toDateTime(1412106600), 'Europe/London') - toRelativeMinuteNum(toDateTime(0), 'Europe/London'); SELECT toRelativeMinuteNum(toDateTime(1412106600), 'Asia/Tokyo') - toRelativeMinuteNum(toDateTime(0), 'Asia/Tokyo'); @@ -302,7 +302,7 @@ SELECT toRelativeMinuteNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelat /* toRelativeSecondNum */ SELECT 'toRelativeSecondNum'; -SELECT toRelativeSecondNum(toDateTime(1412106600), 'Europe/Moscow') - toRelativeSecondNum(toDateTime(0), 'Europe/Moscow'); +SELECT toRelativeSecondNum(toDateTime(1412106600), 'Asia/Istanbul') - toRelativeSecondNum(toDateTime(0), 'Asia/Istanbul'); SELECT toRelativeSecondNum(toDateTime(1412106600), 'Europe/Paris') - toRelativeSecondNum(toDateTime(0), 'Europe/Paris'); SELECT toRelativeSecondNum(toDateTime(1412106600), 'Europe/London') - toRelativeSecondNum(toDateTime(0), 'Europe/London'); SELECT toRelativeSecondNum(toDateTime(1412106600), 'Asia/Tokyo') - toRelativeSecondNum(toDateTime(0), 'Asia/Tokyo'); @@ -311,13 +311,13 @@ SELECT toRelativeSecondNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelat /* toDate */ SELECT 'toDate'; -SELECT toDate(toDateTime(1412106600), 'Europe/Moscow'); +SELECT toDate(toDateTime(1412106600), 'Asia/Istanbul'); SELECT toDate(toDateTime(1412106600), 'Europe/Paris'); SELECT toDate(toDateTime(1412106600), 'Europe/London'); SELECT toDate(toDateTime(1412106600), 'Asia/Tokyo'); SELECT toDate(toDateTime(1412106600), 'Pacific/Pitcairn'); -SELECT toDate(1412106600, 'Europe/Moscow'); +SELECT toDate(1412106600, 'Asia/Istanbul'); SELECT toDate(1412106600, 'Europe/Paris'); SELECT toDate(1412106600, 'Europe/London'); SELECT toDate(1412106600, 'Asia/Tokyo'); @@ -328,7 +328,7 @@ SELECT toDate(16343); /* toString */ SELECT 'toString'; -SELECT toString(toDateTime(1436956200), 'Europe/Moscow'); +SELECT toString(toDateTime(1436956200), 'Asia/Istanbul'); SELECT toString(toDateTime(1436956200), 'Europe/Paris'); SELECT toString(toDateTime(1436956200), 'Europe/London'); SELECT toString(toDateTime(1436956200), 'Asia/Tokyo'); @@ -337,13 +337,13 @@ SELECT toString(toDateTime(1436956200), 'Pacific/Pitcairn'); /* toUnixTimestamp */ SELECT 'toUnixTimestamp'; -SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Europe/Moscow'), 'Europe/Moscow'); -SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Europe/Moscow'), 'Europe/Paris'); -SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Europe/Moscow'), 'Europe/London'); -SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Europe/Moscow'), 'Asia/Tokyo'); -SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Europe/Moscow'), 'Pacific/Pitcairn'); +SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Asia/Istanbul'), 'Asia/Istanbul'); +SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Asia/Istanbul'), 'Europe/Paris'); +SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Asia/Istanbul'), 'Europe/London'); +SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Asia/Istanbul'), 'Asia/Tokyo'); +SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Asia/Istanbul'), 'Pacific/Pitcairn'); -SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Europe/Moscow'), 'Europe/Moscow'); +SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Asia/Istanbul'), 'Asia/Istanbul'); SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Europe/Paris'), 'Europe/Paris'); SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Europe/London'), 'Europe/London'); SELECT toUnixTimestamp(toString(toDateTime(1426415400), 'Asia/Tokyo'), 'Asia/Tokyo'); diff --git a/tests/queries/0_stateless/00206_empty_array_to_single.sql b/tests/queries/0_stateless/00206_empty_array_to_single.sql index 0e3ff4f3537..85e8f82436d 100644 --- a/tests/queries/0_stateless/00206_empty_array_to_single.sql +++ b/tests/queries/0_stateless/00206_empty_array_to_single.sql @@ -1,5 +1,5 @@ SELECT emptyArrayToSingle(arrayFilter(x -> x != 99, arrayJoin([[1, 2], [99], [4, 5, 6]]))); -SELECT emptyArrayToSingle(emptyArrayString()), emptyArrayToSingle(emptyArrayDate()), emptyArrayToSingle(arrayFilter(x -> 0, [now('Europe/Moscow')])); +SELECT emptyArrayToSingle(emptyArrayString()), emptyArrayToSingle(emptyArrayDate()), emptyArrayToSingle(arrayFilter(x -> 0, [now('Asia/Istanbul')])); SELECT emptyArrayToSingle(range(number % 3)), diff --git a/tests/queries/0_stateless/00301_csv.sh b/tests/queries/0_stateless/00301_csv.sh index 50c64b312a7..b2618343dc0 100755 --- a/tests/queries/0_stateless/00301_csv.sh +++ b/tests/queries/0_stateless/00301_csv.sh @@ -18,7 +18,7 @@ Hello "world", 789 ,2016-01-03 $CLICKHOUSE_CLIENT --query="SELECT * FROM csv ORDER BY d, s"; $CLICKHOUSE_CLIENT --query="DROP TABLE csv"; -$CLICKHOUSE_CLIENT --query="CREATE TABLE csv (t DateTime('Europe/Moscow'), s String) ENGINE = Memory"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE csv (t DateTime('Asia/Istanbul'), s String) ENGINE = Memory"; echo '"2016-01-01 01:02:03","1" 2016-01-02 01:02:03, "2" @@ -29,7 +29,7 @@ $CLICKHOUSE_CLIENT --query="SELECT * FROM csv ORDER BY s"; $CLICKHOUSE_CLIENT --query="DROP TABLE csv"; -$CLICKHOUSE_CLIENT --query="CREATE TABLE csv (t Nullable(DateTime('Europe/Moscow')), s Nullable(String)) ENGINE = Memory"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE csv (t Nullable(DateTime('Asia/Istanbul')), s Nullable(String)) ENGINE = Memory"; echo 'NULL, NULL "2016-01-01 01:02:03",NUL diff --git a/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.sh b/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.sh index 91ca0d7d869..f559b9d75bd 100755 --- a/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.sh +++ b/tests/queries/0_stateless/00374_json_each_row_input_with_noisy_fields.sh @@ -26,7 +26,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS json_noisy" echo $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS json_each_row" -$CLICKHOUSE_CLIENT -q "CREATE TABLE json_each_row (d DateTime('Europe/Moscow')) ENGINE = Memory" +$CLICKHOUSE_CLIENT -q "CREATE TABLE json_each_row (d DateTime('Asia/Istanbul')) ENGINE = Memory" echo '{"d" : "2017-08-31 18:36:48", "t" : ""} {"d" : "1504193808", "t" : -1} {"d" : 1504193808, "t" : []} diff --git a/tests/queries/0_stateless/00502_sum_map.sql b/tests/queries/0_stateless/00502_sum_map.sql index 3ceb5b82952..acc87cc5f16 100644 --- a/tests/queries/0_stateless/00502_sum_map.sql +++ b/tests/queries/0_stateless/00502_sum_map.sql @@ -31,7 +31,7 @@ select sumMap(val, cnt) from ( SELECT [ CAST(1, 'UInt64') ] as val, [1] as cnt ) select sumMap(val, cnt) from ( SELECT [ CAST(1, 'Float64') ] as val, [1] as cnt ); select sumMap(val, cnt) from ( SELECT [ CAST('a', 'Enum16(\'a\'=1)') ] as val, [1] as cnt ); -select sumMap(val, cnt) from ( SELECT [ CAST(1, 'DateTime(\'Europe/Moscow\')') ] as val, [1] as cnt ); +select sumMap(val, cnt) from ( SELECT [ CAST(1, 'DateTime(\'Asia/Istanbul\')') ] as val, [1] as cnt ); select sumMap(val, cnt) from ( SELECT [ CAST(1, 'Date') ] as val, [1] as cnt ); select sumMap(val, cnt) from ( SELECT [ CAST('01234567-89ab-cdef-0123-456789abcdef', 'UUID') ] as val, [1] as cnt ); select sumMap(val, cnt) from ( SELECT [ CAST(1.01, 'Decimal(10,2)') ] as val, [1] as cnt ); diff --git a/tests/queries/0_stateless/00506_shard_global_in_union.sql b/tests/queries/0_stateless/00506_shard_global_in_union.sql index b3009add7e5..e51c18c5678 100644 --- a/tests/queries/0_stateless/00506_shard_global_in_union.sql +++ b/tests/queries/0_stateless/00506_shard_global_in_union.sql @@ -22,7 +22,7 @@ DROP TABLE IF EXISTS union_bug; CREATE TABLE union_bug ( Event String, - Datetime DateTime('Europe/Moscow') + Datetime DateTime('Asia/Istanbul') ) Engine = Memory; INSERT INTO union_bug VALUES ('A', 1), ('B', 2); diff --git a/tests/queries/0_stateless/00512_fractional_time_zones.sh b/tests/queries/0_stateless/00512_fractional_time_zones.sh index 45be8fe8d17..eb459d22704 100755 --- a/tests/queries/0_stateless/00512_fractional_time_zones.sh +++ b/tests/queries/0_stateless/00512_fractional_time_zones.sh @@ -4,6 +4,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -TZ=Europe/Moscow ${CLICKHOUSE_LOCAL} --query="SELECT toDateTime('1990-10-19 00:00:00')" +TZ=Asia/Istanbul ${CLICKHOUSE_LOCAL} --query="SELECT toDateTime('1990-10-19 00:00:00')" TZ=Asia/Colombo ${CLICKHOUSE_LOCAL} --query="SELECT toDateTime('1990-10-19 00:00:00')" TZ=Asia/Kathmandu ${CLICKHOUSE_LOCAL} --query="SELECT toDateTime('1990-10-19 00:00:00')" diff --git a/tests/queries/0_stateless/00515_enhanced_time_zones.sql b/tests/queries/0_stateless/00515_enhanced_time_zones.sql index cae487dfab6..e555b6b87bc 100644 --- a/tests/queries/0_stateless/00515_enhanced_time_zones.sql +++ b/tests/queries/0_stateless/00515_enhanced_time_zones.sql @@ -1,6 +1,6 @@ -SELECT addMonths(toDateTime('2017-11-05 08:07:47', 'Europe/Moscow'), 1, 'Asia/Kolkata'); +SELECT addMonths(toDateTime('2017-11-05 08:07:47', 'Asia/Istanbul'), 1, 'Asia/Kolkata'); SELECT addMonths(toDateTime('2017-11-05 10:37:47', 'Asia/Kolkata'), 1); -SELECT addMonths(toTimeZone(toDateTime('2017-11-05 08:07:47', 'Europe/Moscow'), 'Asia/Kolkata'), 1); +SELECT addMonths(toTimeZone(toDateTime('2017-11-05 08:07:47', 'Asia/Istanbul'), 'Asia/Kolkata'), 1); SELECT addMonths(toDateTime('2017-11-05 08:07:47'), 1); SELECT addMonths(materialize(toDateTime('2017-11-05 08:07:47')), 1); @@ -12,11 +12,11 @@ SELECT addMonths(materialize(toDateTime('2017-11-05 08:07:47')), -1); SELECT addMonths(toDateTime('2017-11-05 08:07:47'), materialize(-1)); SELECT addMonths(materialize(toDateTime('2017-11-05 08:07:47')), materialize(-1)); -SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Europe/Moscow'); -SELECT toUnixTimestamp(toDateTime('2017-11-05 08:07:47', 'Europe/Moscow'), 'Europe/Moscow'); +SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Asia/Istanbul'); +SELECT toUnixTimestamp(toDateTime('2017-11-05 08:07:47', 'Asia/Istanbul'), 'Asia/Istanbul'); -SELECT toDateTime('2017-11-05 08:07:47', 'Europe/Moscow'); -SELECT toTimeZone(toDateTime('2017-11-05 08:07:47', 'Europe/Moscow'), 'Asia/Kolkata'); -SELECT toString(toDateTime('2017-11-05 08:07:47', 'Europe/Moscow')); -SELECT toString(toTimeZone(toDateTime('2017-11-05 08:07:47', 'Europe/Moscow'), 'Asia/Kolkata')); -SELECT toString(toDateTime('2017-11-05 08:07:47', 'Europe/Moscow'), 'Asia/Kolkata'); +SELECT toDateTime('2017-11-05 08:07:47', 'Asia/Istanbul'); +SELECT toTimeZone(toDateTime('2017-11-05 08:07:47', 'Asia/Istanbul'), 'Asia/Kolkata'); +SELECT toString(toDateTime('2017-11-05 08:07:47', 'Asia/Istanbul')); +SELECT toString(toTimeZone(toDateTime('2017-11-05 08:07:47', 'Asia/Istanbul'), 'Asia/Kolkata')); +SELECT toString(toDateTime('2017-11-05 08:07:47', 'Asia/Istanbul'), 'Asia/Kolkata'); diff --git a/tests/queries/0_stateless/00538_datediff.sql b/tests/queries/0_stateless/00538_datediff.sql index b76ab4ff3f8..5dc416ad0bd 100644 --- a/tests/queries/0_stateless/00538_datediff.sql +++ b/tests/queries/0_stateless/00538_datediff.sql @@ -43,12 +43,12 @@ SELECT DATEDIFF('year', today(), today() - INTERVAL 10 YEAR); SELECT 'Dependance of timezones'; -SELECT dateDiff('month', toDate('2014-10-26'), toDate('2014-10-27'), 'Europe/Moscow'); -SELECT dateDiff('week', toDate('2014-10-26'), toDate('2014-10-27'), 'Europe/Moscow'); -SELECT dateDiff('day', toDate('2014-10-26'), toDate('2014-10-27'), 'Europe/Moscow'); -SELECT dateDiff('hour', toDate('2014-10-26'), toDate('2014-10-27'), 'Europe/Moscow'); -SELECT dateDiff('minute', toDate('2014-10-26'), toDate('2014-10-27'), 'Europe/Moscow'); -SELECT dateDiff('second', toDate('2014-10-26'), toDate('2014-10-27'), 'Europe/Moscow'); +SELECT dateDiff('month', toDate('2014-10-26'), toDate('2014-10-27'), 'Asia/Istanbul'); +SELECT dateDiff('week', toDate('2014-10-26'), toDate('2014-10-27'), 'Asia/Istanbul'); +SELECT dateDiff('day', toDate('2014-10-26'), toDate('2014-10-27'), 'Asia/Istanbul'); +SELECT dateDiff('hour', toDate('2014-10-26'), toDate('2014-10-27'), 'Asia/Istanbul'); +SELECT dateDiff('minute', toDate('2014-10-26'), toDate('2014-10-27'), 'Asia/Istanbul'); +SELECT dateDiff('second', toDate('2014-10-26'), toDate('2014-10-27'), 'Asia/Istanbul'); SELECT dateDiff('month', toDate('2014-10-26'), toDate('2014-10-27'), 'UTC'); SELECT dateDiff('week', toDate('2014-10-26'), toDate('2014-10-27'), 'UTC'); @@ -57,12 +57,12 @@ SELECT dateDiff('hour', toDate('2014-10-26'), toDate('2014-10-27'), 'UTC'); SELECT dateDiff('minute', toDate('2014-10-26'), toDate('2014-10-27'), 'UTC'); SELECT dateDiff('second', toDate('2014-10-26'), toDate('2014-10-27'), 'UTC'); -SELECT dateDiff('month', toDateTime('2014-10-26 00:00:00', 'Europe/Moscow'), toDateTime('2014-10-27 00:00:00', 'Europe/Moscow')); -SELECT dateDiff('week', toDateTime('2014-10-26 00:00:00', 'Europe/Moscow'), toDateTime('2014-10-27 00:00:00', 'Europe/Moscow')); -SELECT dateDiff('day', toDateTime('2014-10-26 00:00:00', 'Europe/Moscow'), toDateTime('2014-10-27 00:00:00', 'Europe/Moscow')); -SELECT dateDiff('hour', toDateTime('2014-10-26 00:00:00', 'Europe/Moscow'), toDateTime('2014-10-27 00:00:00', 'Europe/Moscow')); -SELECT dateDiff('minute', toDateTime('2014-10-26 00:00:00', 'Europe/Moscow'), toDateTime('2014-10-27 00:00:00', 'Europe/Moscow')); -SELECT dateDiff('second', toDateTime('2014-10-26 00:00:00', 'Europe/Moscow'), toDateTime('2014-10-27 00:00:00', 'Europe/Moscow')); +SELECT dateDiff('month', toDateTime('2014-10-26 00:00:00', 'Asia/Istanbul'), toDateTime('2014-10-27 00:00:00', 'Asia/Istanbul')); +SELECT dateDiff('week', toDateTime('2014-10-26 00:00:00', 'Asia/Istanbul'), toDateTime('2014-10-27 00:00:00', 'Asia/Istanbul')); +SELECT dateDiff('day', toDateTime('2014-10-26 00:00:00', 'Asia/Istanbul'), toDateTime('2014-10-27 00:00:00', 'Asia/Istanbul')); +SELECT dateDiff('hour', toDateTime('2014-10-26 00:00:00', 'Asia/Istanbul'), toDateTime('2014-10-27 00:00:00', 'Asia/Istanbul')); +SELECT dateDiff('minute', toDateTime('2014-10-26 00:00:00', 'Asia/Istanbul'), toDateTime('2014-10-27 00:00:00', 'Asia/Istanbul')); +SELECT dateDiff('second', toDateTime('2014-10-26 00:00:00', 'Asia/Istanbul'), toDateTime('2014-10-27 00:00:00', 'Asia/Istanbul')); SELECT dateDiff('month', toDateTime('2014-10-26 00:00:00', 'UTC'), toDateTime('2014-10-27 00:00:00', 'UTC')); SELECT dateDiff('week', toDateTime('2014-10-26 00:00:00', 'UTC'), toDateTime('2014-10-27 00:00:00', 'UTC')); diff --git a/tests/queries/0_stateless/00561_storage_join.sql b/tests/queries/0_stateless/00561_storage_join.sql index ad4fab09c20..913ecec6f4a 100644 --- a/tests/queries/0_stateless/00561_storage_join.sql +++ b/tests/queries/0_stateless/00561_storage_join.sql @@ -1,7 +1,7 @@ drop table IF EXISTS joinbug; CREATE TABLE joinbug ( - event_date Date MATERIALIZED toDate(created, 'Europe/Moscow'), + event_date Date MATERIALIZED toDate(created, 'Asia/Istanbul'), id UInt64, id2 UInt64, val UInt64, diff --git a/tests/queries/0_stateless/00718_format_datetime.sql b/tests/queries/0_stateless/00718_format_datetime.sql index cd679fe9735..899e23f9da1 100644 --- a/tests/queries/0_stateless/00718_format_datetime.sql +++ b/tests/queries/0_stateless/00718_format_datetime.sql @@ -42,4 +42,4 @@ SELECT formatDateTime(toDateTime('2018-01-02 22:33:44'), 'no formatting pattern' SELECT formatDateTime(toDate('2018-01-01'), '%F %T'); SELECT formatDateTime(toDateTime('2018-01-01 01:00:00', 'UTC'), '%F %T', 'UTC'), - formatDateTime(toDateTime('2018-01-01 01:00:00', 'UTC'), '%F %T', 'Europe/Moscow') \ No newline at end of file + formatDateTime(toDateTime('2018-01-01 01:00:00', 'UTC'), '%F %T', 'Asia/Istanbul') \ No newline at end of file diff --git a/tests/queries/0_stateless/00719_format_datetime_rand.sql b/tests/queries/0_stateless/00719_format_datetime_rand.sql index b91c988e1ca..ee596912bc7 100644 --- a/tests/queries/0_stateless/00719_format_datetime_rand.sql +++ b/tests/queries/0_stateless/00719_format_datetime_rand.sql @@ -8,4 +8,4 @@ WITH toDate(today() + rand() % 4096) AS t SELECT count() FROM numbers(1000000) W -- Note: in some other timezones, daylight saving time change happens in midnight, so the first time of day is 01:00:00 instead of 00:00:00. -- Stick to Moscow timezone to avoid this issue. -WITH toDate(today() + rand() % 4096) AS t SELECT count() FROM numbers(1000000) WHERE formatDateTime(t, '%F %T', 'Europe/Moscow') != toString(toDateTime(t, 'Europe/Moscow')); +WITH toDate(today() + rand() % 4096) AS t SELECT count() FROM numbers(1000000) WHERE formatDateTime(t, '%F %T', 'Asia/Istanbul') != toString(toDateTime(t, 'Asia/Istanbul')); diff --git a/tests/queries/0_stateless/00735_long_conditional.reference b/tests/queries/0_stateless/00735_long_conditional.reference index 082c2d49de9..f6c06e64066 100644 --- a/tests/queries/0_stateless/00735_long_conditional.reference +++ b/tests/queries/0_stateless/00735_long_conditional.reference @@ -92,9 +92,9 @@ value vs value 0 1 1 UInt64 Decimal(18, 0) Decimal(38, 0) 0 1 1 UInt64 Decimal(38, 0) Decimal(38, 0) 1970-01-01 1970-01-02 1970-01-02 Date Date Date -2000-01-01 2000-01-01 00:00:01 2000-01-01 00:00:01 Date DateTime(\'Europe/Moscow\') DateTime(\'Europe/Moscow\') -2000-01-01 00:00:00 2000-01-02 2000-01-02 00:00:00 DateTime(\'Europe/Moscow\') Date DateTime(\'Europe/Moscow\') -1970-01-01 03:00:00 1970-01-01 03:00:01 1970-01-01 03:00:01 DateTime(\'Europe/Moscow\') DateTime(\'Europe/Moscow\') DateTime(\'Europe/Moscow\') +2000-01-01 2000-01-01 00:00:01 2000-01-01 00:00:01 Date DateTime(\'Asia/Istanbul\') DateTime(\'Asia/Istanbul\') +2000-01-01 00:00:00 2000-01-02 2000-01-02 00:00:00 DateTime(\'Asia/Istanbul\') Date DateTime(\'Asia/Istanbul\') +1970-01-01 03:00:00 1970-01-01 03:00:01 1970-01-01 03:00:01 DateTime(\'Asia/Istanbul\') DateTime(\'Asia/Istanbul\') DateTime(\'Asia/Istanbul\') column vs value 0 1 1 Int8 Int8 Int8 0 1 1 Int8 Int16 Int16 @@ -189,6 +189,6 @@ column vs value 0 1 1 UInt64 Decimal(18, 0) Decimal(38, 0) 0 1 1 UInt64 Decimal(38, 0) Decimal(38, 0) 1970-01-01 1970-01-02 1970-01-02 Date Date Date -2000-01-01 2000-01-01 00:00:01 2000-01-01 00:00:01 Date DateTime(\'Europe/Moscow\') DateTime(\'Europe/Moscow\') -2000-01-01 00:00:00 2000-01-02 2000-01-02 00:00:00 DateTime(\'Europe/Moscow\') Date DateTime(\'Europe/Moscow\') -1970-01-01 03:00:00 1970-01-01 03:00:01 1970-01-01 03:00:01 DateTime(\'Europe/Moscow\') DateTime(\'Europe/Moscow\') DateTime(\'Europe/Moscow\') +2000-01-01 2000-01-01 00:00:01 2000-01-01 00:00:01 Date DateTime(\'Asia/Istanbul\') DateTime(\'Asia/Istanbul\') +2000-01-01 00:00:00 2000-01-02 2000-01-02 00:00:00 DateTime(\'Asia/Istanbul\') Date DateTime(\'Asia/Istanbul\') +1970-01-01 03:00:00 1970-01-01 03:00:01 1970-01-01 03:00:01 DateTime(\'Asia/Istanbul\') DateTime(\'Asia/Istanbul\') DateTime(\'Asia/Istanbul\') diff --git a/tests/queries/0_stateless/00735_long_conditional.sql b/tests/queries/0_stateless/00735_long_conditional.sql index 73febad1310..662c87db48f 100644 --- a/tests/queries/0_stateless/00735_long_conditional.sql +++ b/tests/queries/0_stateless/00735_long_conditional.sql @@ -15,7 +15,7 @@ SELECT toInt8(0) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), SELECT toInt8(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt8(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt8(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT toInt8(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toInt8(0) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt8(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt8(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt8(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -31,7 +31,7 @@ SELECT toInt16(0) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), SELECT toInt16(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt16(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt16(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT toInt16(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toInt16(0) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt16(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt16(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt16(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -47,7 +47,7 @@ SELECT toInt32(0) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), SELECT toInt32(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt32(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt32(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT toInt32(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toInt32(0) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt32(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt32(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt32(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -63,7 +63,7 @@ SELECT toInt64(0) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), SELECT toInt64(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt64(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt64(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT toInt64(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toInt64(0) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt64(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt64(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt64(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -79,7 +79,7 @@ SELECT toUInt8(0) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), SELECT toUInt8(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt8(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt8(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT toUInt8(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toUInt8(0) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt8(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt8(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt8(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -95,7 +95,7 @@ SELECT toUInt16(0) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x) SELECT toUInt16(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt16(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt16(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT toUInt16(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toUInt16(0) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt16(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt16(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt16(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -111,7 +111,7 @@ SELECT toUInt32(0) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x) SELECT toUInt32(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt32(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt32(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT toUInt32(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toUInt32(0) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt32(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt32(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt32(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -127,7 +127,7 @@ SELECT toUInt64(0) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x) SELECT toUInt64(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt64(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt64(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT toUInt64(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toUInt64(0) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt64(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt64(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt64(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -143,26 +143,26 @@ SELECT toDate(0) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), SELECT toDate(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toDate(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toDate(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -SELECT toDate('2000-01-01') AS x, toDateTime('2000-01-01 00:00:01', 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toDate('2000-01-01') AS x, toDateTime('2000-01-01 00:00:01', 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toDate(0) AS x, toDecimal32(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toDate(0) AS x, toDecimal64(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toDate(0) AS x, toDecimal128(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT toDateTime(0, 'Europe/Moscow') AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toDateTime(0, 'Europe/Moscow') AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toDateTime(0, 'Europe/Moscow') AS x, toInt32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toDateTime(0, 'Europe/Moscow') AS x, toInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toDateTime(0, 'Europe/Moscow') AS x, toUInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toDateTime(0, 'Europe/Moscow') AS x, toUInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toDateTime(0, 'Europe/Moscow') AS x, toUInt32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toDateTime(0, 'Europe/Moscow') AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toDateTime(0, 'Europe/Moscow') AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toDateTime(0, 'Europe/Moscow') AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toDateTime('2000-01-01 00:00:00', 'Europe/Moscow') AS x, toDate('2000-01-02') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -SELECT toDateTime(0, 'Europe/Moscow') AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -SELECT toDateTime(0, 'Europe/Moscow') AS x, toDecimal32(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toDateTime(0, 'Europe/Moscow') AS x, toDecimal64(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toDateTime(0, 'Europe/Moscow') AS x, toDecimal128(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toInt32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toUInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toUInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toUInt32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul') AS x, toDate('2000-01-02') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toDecimal32(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toDecimal64(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT toDateTime(0, 'Asia/Istanbul') AS x, toDecimal128(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT 'column vs value'; @@ -177,7 +177,7 @@ SELECT materialize(toInt8(0)) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, to SELECT materialize(toInt8(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt8(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt8(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toInt8(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toInt8(0)) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt8(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt8(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt8(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -193,7 +193,7 @@ SELECT materialize(toInt16(0)) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, t SELECT materialize(toInt16(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt16(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt16(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toInt16(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toInt16(0)) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt16(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt16(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt16(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -209,7 +209,7 @@ SELECT materialize(toInt32(0)) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, t SELECT materialize(toInt32(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt32(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt32(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toInt32(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toInt32(0)) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt32(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt32(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt32(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -225,7 +225,7 @@ SELECT materialize(toInt64(0)) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, t SELECT materialize(toInt64(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt64(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt64(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toInt64(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toInt64(0)) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt64(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt64(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt64(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -241,7 +241,7 @@ SELECT materialize(toUInt8(0)) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, t SELECT materialize(toUInt8(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt8(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt8(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toUInt8(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toUInt8(0)) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt8(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt8(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt8(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -257,7 +257,7 @@ SELECT materialize(toUInt16(0)) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt16(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt16(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt16(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toUInt16(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toUInt16(0)) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt16(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt16(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt16(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -273,7 +273,7 @@ SELECT materialize(toUInt32(0)) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt32(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt32(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt32(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toUInt32(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toUInt32(0)) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt32(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt32(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt32(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -289,7 +289,7 @@ SELECT materialize(toUInt64(0)) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt64(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt64(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt64(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toUInt64(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toUInt64(0)) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt64(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt64(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt64(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -305,23 +305,23 @@ SELECT materialize(toDate(0)) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, to SELECT materialize(toDate(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toDate(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toDate(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -SELECT materialize(toDate('2000-01-01')) AS x, toDateTime('2000-01-01 00:00:01', 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toDate('2000-01-01')) AS x, toDateTime('2000-01-01 00:00:01', 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toDate(0)) AS x, toDecimal32(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toDate(0)) AS x, toDecimal64(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toDate(0)) AS x, toDecimal128(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toInt32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toUInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toUInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toUInt32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Europe/Moscow')) AS x, toDate('2000-01-02') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toDecimal32(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toDecimal64(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toDecimal128(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toInt32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toUInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toUInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toUInt32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul')) AS x, toDate('2000-01-02') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toDateTime(1, 'Asia/Istanbul') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toDecimal32(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toDecimal64(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } +SELECT materialize(toDateTime(0, 'Asia/Istanbul')) AS x, toDecimal128(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } diff --git a/tests/queries/0_stateless/00801_daylight_saving_time_hour_underflow.sql b/tests/queries/0_stateless/00801_daylight_saving_time_hour_underflow.sql index fa98d087a62..a86a863124c 100644 --- a/tests/queries/0_stateless/00801_daylight_saving_time_hour_underflow.sql +++ b/tests/queries/0_stateless/00801_daylight_saving_time_hour_underflow.sql @@ -1,6 +1,6 @@ -- See comment in DateLUTImpl.cpp: "We doesn't support cases when time change results in switching to previous day..." SELECT - ignore(toDateTime(370641600, 'Europe/Moscow') AS t), + ignore(toDateTime(370641600, 'Asia/Istanbul') AS t), replaceRegexpAll(toString(t), '\\d', 'x'), toHour(t) < 24, replaceRegexpAll(formatDateTime(t, '%Y-%m-%d %H:%M:%S; %R:%S; %F %T'), '\\d', 'x'); diff --git a/tests/queries/0_stateless/00802_daylight_saving_time_shift_backwards_at_midnight.sql b/tests/queries/0_stateless/00802_daylight_saving_time_shift_backwards_at_midnight.sql index 4244ce2039b..1fdd9b20b2b 100644 --- a/tests/queries/0_stateless/00802_daylight_saving_time_shift_backwards_at_midnight.sql +++ b/tests/queries/0_stateless/00802_daylight_saving_time_shift_backwards_at_midnight.sql @@ -1,3 +1,3 @@ -- concat with empty string to defeat injectiveness of toString assumption. -SELECT concat('', toString(toDateTime('1981-09-29 00:00:00', 'Europe/Moscow') + INTERVAL number * 300 SECOND)) AS k FROM numbers(10000) GROUP BY k HAVING count() > 1 ORDER BY k; +SELECT concat('', toString(toDateTime('1981-09-29 00:00:00', 'Asia/Istanbul') + INTERVAL number * 300 SECOND)) AS k FROM numbers(10000) GROUP BY k HAVING count() > 1 ORDER BY k; SELECT concat('', toString(toDateTime('2018-09-19 00:00:00', 'Asia/Tehran') + INTERVAL number * 300 SECOND)) AS k FROM numbers(1000) GROUP BY k HAVING count() > 1 ORDER BY k; diff --git a/tests/queries/0_stateless/00825_protobuf_format_persons.sh b/tests/queries/0_stateless/00825_protobuf_format_persons.sh index bb376e6ed70..fe53e2995f9 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_persons.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_persons.sh @@ -28,7 +28,7 @@ CREATE TABLE persons_00825 (uuid UUID, photo Nullable(String), phoneNumber Nullable(FixedString(13)), isOnline UInt8, - visitTime Nullable(DateTime('Europe/Moscow')), + visitTime Nullable(DateTime('Asia/Istanbul')), age UInt8, zodiacSign Enum16('aries'=321, 'taurus'=420, 'gemini'=521, 'cancer'=621, 'leo'=723, 'virgo'=823, 'libra'=923, 'scorpius'=1023, 'sagittarius'=1122, 'capricorn'=1222, 'aquarius'=120, @@ -47,9 +47,9 @@ CREATE TABLE persons_00825 (uuid UUID, "nestiness_a_B.c_E" Array(UInt32) ) ENGINE = MergeTree ORDER BY tuple(); -INSERT INTO persons_00825 VALUES (toUUID('a7522158-3d41-4b77-ad69-6c598ee55c49'), 'Ivan', 'Petrov', 'male', toDate('1980-12-29'), 'png', '+74951234567', 1, toDateTime('2019-01-05 18:45:00', 'Europe/Moscow'), 38, 'capricorn', ['Yesterday', 'Flowers'], [255, 0, 0], 'Moscow', [55.753215, 37.622504], 3.14, 214.10, 0.1, 5.8, 17060000000, ['meter', 'centimeter', 'kilometer'], [1, 0.01, 1000], 500, [501, 502]); +INSERT INTO persons_00825 VALUES (toUUID('a7522158-3d41-4b77-ad69-6c598ee55c49'), 'Ivan', 'Petrov', 'male', toDate('1980-12-29'), 'png', '+74951234567', 1, toDateTime('2019-01-05 18:45:00', 'Asia/Istanbul'), 38, 'capricorn', ['Yesterday', 'Flowers'], [255, 0, 0], 'Moscow', [55.753215, 37.622504], 3.14, 214.10, 0.1, 5.8, 17060000000, ['meter', 'centimeter', 'kilometer'], [1, 0.01, 1000], 500, [501, 502]); INSERT INTO persons_00825 VALUES (toUUID('c694ad8a-f714-4ea3-907d-fd54fb25d9b5'), 'Natalia', 'Sokolova', 'female', toDate('1992-03-08'), 'jpg', NULL, 0, NULL, 26, 'pisces', [], [100, 200, 50], 'Plymouth', [50.403724, -4.142123], 3.14159, NULL, 0.007, 5.4, -20000000000000, [], [], NULL, []); -INSERT INTO persons_00825 VALUES (toUUID('a7da1aa6-f425-4789-8947-b034786ed374'), 'Vasily', 'Sidorov', 'male', toDate('1995-07-28'), 'bmp', '+442012345678', 1, toDateTime('2018-12-30 00:00:00', 'Europe/Moscow'), 23, 'leo', ['Sunny'], [250, 244, 10], 'Murmansk', [68.970682, 33.074981], 3.14159265358979, 100000000000, 800, -3.2, 154400000, ['pound'], [16], 503, []); +INSERT INTO persons_00825 VALUES (toUUID('a7da1aa6-f425-4789-8947-b034786ed374'), 'Vasily', 'Sidorov', 'male', toDate('1995-07-28'), 'bmp', '+442012345678', 1, toDateTime('2018-12-30 00:00:00', 'Asia/Istanbul'), 23, 'leo', ['Sunny'], [250, 244, 10], 'Murmansk', [68.970682, 33.074981], 3.14159265358979, 100000000000, 800, -3.2, 154400000, ['pound'], [16], 503, []); SELECT * FROM persons_00825 ORDER BY name; EOF diff --git a/tests/queries/0_stateless/00835_if_generic_case.sql b/tests/queries/0_stateless/00835_if_generic_case.sql index 011cea46ffc..63baffcf17d 100644 --- a/tests/queries/0_stateless/00835_if_generic_case.sql +++ b/tests/queries/0_stateless/00835_if_generic_case.sql @@ -1,17 +1,17 @@ -SELECT toDateTime('2000-01-01 00:00:00', 'Europe/Moscow') AS x, toDate('2000-01-02') AS y, x > y ? x : y AS z; -SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Europe/Moscow')) AS x, toDate('2000-01-02') AS y, x > y ? x : y AS z; -SELECT toDateTime('2000-01-01 00:00:00', 'Europe/Moscow') AS x, materialize(toDate('2000-01-02')) AS y, x > y ? x : y AS z; -SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Europe/Moscow')) AS x, materialize(toDate('2000-01-02')) AS y, x > y ? x : y AS z; +SELECT toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul') AS x, toDate('2000-01-02') AS y, x > y ? x : y AS z; +SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul')) AS x, toDate('2000-01-02') AS y, x > y ? x : y AS z; +SELECT toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul') AS x, materialize(toDate('2000-01-02')) AS y, x > y ? x : y AS z; +SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul')) AS x, materialize(toDate('2000-01-02')) AS y, x > y ? x : y AS z; -SELECT toDateTime('2000-01-01 00:00:00', 'Europe/Moscow') AS x, toDate('2000-01-02') AS y, 0 ? x : y AS z; -SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Europe/Moscow')) AS x, toDate('2000-01-02') AS y, 0 ? x : y AS z; -SELECT toDateTime('2000-01-01 00:00:00', 'Europe/Moscow') AS x, materialize(toDate('2000-01-02')) AS y, 0 ? x : y AS z; -SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Europe/Moscow')) AS x, materialize(toDate('2000-01-02')) AS y, 0 ? x : y AS z; +SELECT toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul') AS x, toDate('2000-01-02') AS y, 0 ? x : y AS z; +SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul')) AS x, toDate('2000-01-02') AS y, 0 ? x : y AS z; +SELECT toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul') AS x, materialize(toDate('2000-01-02')) AS y, 0 ? x : y AS z; +SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul')) AS x, materialize(toDate('2000-01-02')) AS y, 0 ? x : y AS z; -SELECT toDateTime('2000-01-01 00:00:00', 'Europe/Moscow') AS x, toDate('2000-01-02') AS y, 1 ? x : y AS z; -SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Europe/Moscow')) AS x, toDate('2000-01-02') AS y, 1 ? x : y AS z; -SELECT toDateTime('2000-01-01 00:00:00', 'Europe/Moscow') AS x, materialize(toDate('2000-01-02')) AS y, 1 ? x : y AS z; -SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Europe/Moscow')) AS x, materialize(toDate('2000-01-02')) AS y, 1 ? x : y AS z; +SELECT toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul') AS x, toDate('2000-01-02') AS y, 1 ? x : y AS z; +SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul')) AS x, toDate('2000-01-02') AS y, 1 ? x : y AS z; +SELECT toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul') AS x, materialize(toDate('2000-01-02')) AS y, 1 ? x : y AS z; +SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul')) AS x, materialize(toDate('2000-01-02')) AS y, 1 ? x : y AS z; SELECT rand() % 2 = 0 ? number : number FROM numbers(5); diff --git a/tests/queries/0_stateless/00910_decimal_group_array_crash_3783.sql b/tests/queries/0_stateless/00910_decimal_group_array_crash_3783.sql index cf0e0bac3dd..c6151d3bdee 100644 --- a/tests/queries/0_stateless/00910_decimal_group_array_crash_3783.sql +++ b/tests/queries/0_stateless/00910_decimal_group_array_crash_3783.sql @@ -8,7 +8,7 @@ select groupArray(s) from (select sum(n) s from (select toDecimal128(number, 10) DROP TABLE IF EXISTS sensor_value; CREATE TABLE sensor_value ( - received_at DateTime('Europe/Moscow'), + received_at DateTime('Asia/Istanbul'), device_id UUID, sensor_id UUID, value Nullable(Decimal(18, 4)), @@ -35,28 +35,28 @@ DROP TABLE sensor_value; select s.a, s.b, max(s.dt1) dt1, s.c, s.d, s.f, s.i, max(s.dt2) dt2 from ( select toUInt64(4360430) a , toUInt64(5681495) b - , toDateTime('2018-11-01 10:44:58', 'Europe/Moscow') dt1 + , toDateTime('2018-11-01 10:44:58', 'Asia/Istanbul') dt1 , 'txt' c , toDecimal128('274.350000000000', 12) d , toDecimal128(268.970000000000, 12) f , toDecimal128(0.000000000000, 12) i - , toDateTime('2018-11-02 00:00:00', 'Europe/Moscow') dt2 + , toDateTime('2018-11-02 00:00:00', 'Asia/Istanbul') dt2 union all select toUInt64(4341757) a , toUInt64(5657967) b - , toDateTime('2018-11-01 16:47:46', 'Europe/Moscow') dt1 + , toDateTime('2018-11-01 16:47:46', 'Asia/Istanbul') dt1 , 'txt' c , toDecimal128('321.380000000000', 12) d , toDecimal128(315.080000000000, 12) f , toDecimal128(0.000000000000, 12) i - , toDateTime('2018-11-02 00:00:00', 'Europe/Moscow') dt2 + , toDateTime('2018-11-02 00:00:00', 'Asia/Istanbul') dt2 union all select toUInt64(4360430) a , toUInt64(5681495) b - , toDateTime('2018-11-02 09:00:07', 'Europe/Moscow') dt1 + , toDateTime('2018-11-02 09:00:07', 'Asia/Istanbul') dt1 , 'txt' c , toDecimal128('274.350000000000', 12) d , toDecimal128(268.970000000000, 12) f , toDecimal128(0.000000000000, 12) i - , toDateTime('2018-11-02 00:00:00', 'Europe/Moscow') dt2 + , toDateTime('2018-11-02 00:00:00', 'Asia/Istanbul') dt2 ) s group by s.a, s.b, s.c, s.d, s.f, s.i ORDER BY s.a, s.b, s.c, s.d, s.f, s.i; diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility_long.python b/tests/queries/0_stateless/00921_datetime64_compatibility_long.python index 03cc088fd87..e4dbc716d7e 100644 --- a/tests/queries/0_stateless/00921_datetime64_compatibility_long.python +++ b/tests/queries/0_stateless/00921_datetime64_compatibility_long.python @@ -9,69 +9,69 @@ import argparse # Create SQL statement to verify dateTime64 is accepted as argument to functions taking DateTime. FUNCTIONS=""" toTimeZone(N, 'UTC') -toYear(N, 'Europe/Moscow') -toQuarter(N, 'Europe/Moscow') -toMonth(N, 'Europe/Moscow') -toDayOfYear(N, 'Europe/Moscow') -toDayOfMonth(N, 'Europe/Moscow') -toDayOfWeek(N, 'Europe/Moscow') -toHour(N, 'Europe/Moscow') -toMinute(N, 'Europe/Moscow') -toSecond(N, 'Europe/Moscow') +toYear(N, 'Asia/Istanbul') +toQuarter(N, 'Asia/Istanbul') +toMonth(N, 'Asia/Istanbul') +toDayOfYear(N, 'Asia/Istanbul') +toDayOfMonth(N, 'Asia/Istanbul') +toDayOfWeek(N, 'Asia/Istanbul') +toHour(N, 'Asia/Istanbul') +toMinute(N, 'Asia/Istanbul') +toSecond(N, 'Asia/Istanbul') toUnixTimestamp(N) -toStartOfYear(N, 'Europe/Moscow') -toStartOfISOYear(N, 'Europe/Moscow') -toStartOfQuarter(N, 'Europe/Moscow') -toStartOfMonth(N, 'Europe/Moscow') -toMonday(N, 'Europe/Moscow') -toStartOfWeek(N, 'Europe/Moscow') -toStartOfDay(N, 'Europe/Moscow') -toStartOfHour(N, 'Europe/Moscow') -toStartOfMinute(N, 'Europe/Moscow') -toStartOfFiveMinute(N, 'Europe/Moscow') -toStartOfTenMinutes(N, 'Europe/Moscow') -toStartOfFifteenMinutes(N, 'Europe/Moscow') -toStartOfInterval(N, INTERVAL 1 year, 'Europe/Moscow') -toStartOfInterval(N, INTERVAL 1 month, 'Europe/Moscow') -toStartOfInterval(N, INTERVAL 1 day, 'Europe/Moscow') -toStartOfInterval(N, INTERVAL 15 minute, 'Europe/Moscow') -date_trunc('year', N, 'Europe/Moscow') -date_trunc('month', N, 'Europe/Moscow') -date_trunc('day', N, 'Europe/Moscow') -date_trunc('minute', N, 'Europe/Moscow') -toTime(N, 'Europe/Moscow') -toRelativeYearNum(N, 'Europe/Moscow') -toRelativeQuarterNum(N, 'Europe/Moscow') -toRelativeMonthNum(N, 'Europe/Moscow') -toRelativeWeekNum(N, 'Europe/Moscow') -toRelativeDayNum(N, 'Europe/Moscow') -toRelativeHourNum(N, 'Europe/Moscow') -toRelativeMinuteNum(N, 'Europe/Moscow') -toRelativeSecondNum(N, 'Europe/Moscow') -toISOYear(N, 'Europe/Moscow') -toISOWeek(N, 'Europe/Moscow') -toWeek(N, 'Europe/Moscow') -toYearWeek(N, 'Europe/Moscow') -timeSlot(N, 'Europe/Moscow') -toYYYYMM(N, 'Europe/Moscow') -toYYYYMMDD(N, 'Europe/Moscow') -toYYYYMMDDhhmmss(N, 'Europe/Moscow') -addYears(N, 1, 'Europe/Moscow') -addMonths(N, 1, 'Europe/Moscow') -addWeeks(N, 1, 'Europe/Moscow') -addDays(N, 1, 'Europe/Moscow') -addHours(N, 1, 'Europe/Moscow') -addMinutes(N, 1, 'Europe/Moscow') -addSeconds(N, 1, 'Europe/Moscow') -addQuarters(N, 1, 'Europe/Moscow') -subtractYears(N, 1, 'Europe/Moscow') -subtractMonths(N, 1, 'Europe/Moscow') -subtractWeeks(N, 1, 'Europe/Moscow') -subtractDays(N, 1, 'Europe/Moscow') -subtractHours(N, 1, 'Europe/Moscow') -subtractMinutes(N, 1, 'Europe/Moscow') -subtractSeconds(N, 1, 'Europe/Moscow') -subtractQuarters(N, 1, 'Europe/Moscow') +toStartOfYear(N, 'Asia/Istanbul') +toStartOfISOYear(N, 'Asia/Istanbul') +toStartOfQuarter(N, 'Asia/Istanbul') +toStartOfMonth(N, 'Asia/Istanbul') +toMonday(N, 'Asia/Istanbul') +toStartOfWeek(N, 'Asia/Istanbul') +toStartOfDay(N, 'Asia/Istanbul') +toStartOfHour(N, 'Asia/Istanbul') +toStartOfMinute(N, 'Asia/Istanbul') +toStartOfFiveMinute(N, 'Asia/Istanbul') +toStartOfTenMinutes(N, 'Asia/Istanbul') +toStartOfFifteenMinutes(N, 'Asia/Istanbul') +toStartOfInterval(N, INTERVAL 1 year, 'Asia/Istanbul') +toStartOfInterval(N, INTERVAL 1 month, 'Asia/Istanbul') +toStartOfInterval(N, INTERVAL 1 day, 'Asia/Istanbul') +toStartOfInterval(N, INTERVAL 15 minute, 'Asia/Istanbul') +date_trunc('year', N, 'Asia/Istanbul') +date_trunc('month', N, 'Asia/Istanbul') +date_trunc('day', N, 'Asia/Istanbul') +date_trunc('minute', N, 'Asia/Istanbul') +toTime(N, 'Asia/Istanbul') +toRelativeYearNum(N, 'Asia/Istanbul') +toRelativeQuarterNum(N, 'Asia/Istanbul') +toRelativeMonthNum(N, 'Asia/Istanbul') +toRelativeWeekNum(N, 'Asia/Istanbul') +toRelativeDayNum(N, 'Asia/Istanbul') +toRelativeHourNum(N, 'Asia/Istanbul') +toRelativeMinuteNum(N, 'Asia/Istanbul') +toRelativeSecondNum(N, 'Asia/Istanbul') +toISOYear(N, 'Asia/Istanbul') +toISOWeek(N, 'Asia/Istanbul') +toWeek(N, 'Asia/Istanbul') +toYearWeek(N, 'Asia/Istanbul') +timeSlot(N, 'Asia/Istanbul') +toYYYYMM(N, 'Asia/Istanbul') +toYYYYMMDD(N, 'Asia/Istanbul') +toYYYYMMDDhhmmss(N, 'Asia/Istanbul') +addYears(N, 1, 'Asia/Istanbul') +addMonths(N, 1, 'Asia/Istanbul') +addWeeks(N, 1, 'Asia/Istanbul') +addDays(N, 1, 'Asia/Istanbul') +addHours(N, 1, 'Asia/Istanbul') +addMinutes(N, 1, 'Asia/Istanbul') +addSeconds(N, 1, 'Asia/Istanbul') +addQuarters(N, 1, 'Asia/Istanbul') +subtractYears(N, 1, 'Asia/Istanbul') +subtractMonths(N, 1, 'Asia/Istanbul') +subtractWeeks(N, 1, 'Asia/Istanbul') +subtractDays(N, 1, 'Asia/Istanbul') +subtractHours(N, 1, 'Asia/Istanbul') +subtractMinutes(N, 1, 'Asia/Istanbul') +subtractSeconds(N, 1, 'Asia/Istanbul') +subtractQuarters(N, 1, 'Asia/Istanbul') CAST(N as DateTime('Europe/Minsk')) CAST(N as Date) CAST(N as UInt64) @@ -80,10 +80,10 @@ CAST(N as DateTime64(3, 'Europe/Minsk')) CAST(N as DateTime64(6, 'Europe/Minsk')) CAST(N as DateTime64(9, 'Europe/Minsk')) # Casting our test values to DateTime(12) will cause an overflow and hence will fail the test under UB sanitizer. -# CAST(N as DateTime64(12, 'Europe/Moscow')) +# CAST(N as DateTime64(12, 'Asia/Istanbul')) # DateTime64(18) will always fail due to zero precision, but it is Ok to test here: -# CAST(N as DateTime64(18, 'Europe/Moscow')) -formatDateTime(N, '%C %d %D %e %F %H %I %j %m %M %p %R %S %T %u %V %w %y %Y %%', 'Europe/Moscow') +# CAST(N as DateTime64(18, 'Asia/Istanbul')) +formatDateTime(N, '%C %d %D %e %F %H %I %j %m %M %p %R %S %T %u %V %w %y %Y %%', 'Asia/Istanbul') """.splitlines() # Expanded later to cartesian product of all arguments, using format string. diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference b/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference index 230bfa0c117..fa21fae0121 100644 --- a/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference +++ b/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference @@ -3,47 +3,47 @@ Code: 43 "DateTime('UTC')","2019-09-16 16:20:11" "DateTime64(3, 'UTC')","2019-09-16 16:20:11.234" ------------------------------------------ -SELECT toYear(N, \'Europe/Moscow\') +SELECT toYear(N, \'Asia/Istanbul\') "UInt16",2019 "UInt16",2019 "UInt16",2019 ------------------------------------------ -SELECT toQuarter(N, \'Europe/Moscow\') +SELECT toQuarter(N, \'Asia/Istanbul\') "UInt8",3 "UInt8",3 "UInt8",3 ------------------------------------------ -SELECT toMonth(N, \'Europe/Moscow\') +SELECT toMonth(N, \'Asia/Istanbul\') "UInt8",9 "UInt8",9 "UInt8",9 ------------------------------------------ -SELECT toDayOfYear(N, \'Europe/Moscow\') +SELECT toDayOfYear(N, \'Asia/Istanbul\') "UInt16",259 "UInt16",259 "UInt16",259 ------------------------------------------ -SELECT toDayOfMonth(N, \'Europe/Moscow\') +SELECT toDayOfMonth(N, \'Asia/Istanbul\') "UInt8",16 "UInt8",16 "UInt8",16 ------------------------------------------ -SELECT toDayOfWeek(N, \'Europe/Moscow\') +SELECT toDayOfWeek(N, \'Asia/Istanbul\') "UInt8",1 "UInt8",1 "UInt8",1 ------------------------------------------ -SELECT toHour(N, \'Europe/Moscow\') +SELECT toHour(N, \'Asia/Istanbul\') Code: 43 "UInt8",19 "UInt8",19 ------------------------------------------ -SELECT toMinute(N, \'Europe/Moscow\') +SELECT toMinute(N, \'Asia/Istanbul\') Code: 43 "UInt8",20 "UInt8",20 ------------------------------------------ -SELECT toSecond(N, \'Europe/Moscow\') +SELECT toSecond(N, \'Asia/Istanbul\') Code: 43 "UInt8",11 "UInt8",11 @@ -53,269 +53,269 @@ Code: 44 "UInt32",1568650811 "UInt32",1568650811 ------------------------------------------ -SELECT toStartOfYear(N, \'Europe/Moscow\') +SELECT toStartOfYear(N, \'Asia/Istanbul\') Code: 43 "Date","2019-01-01" "Date","2019-01-01" ------------------------------------------ -SELECT toStartOfISOYear(N, \'Europe/Moscow\') +SELECT toStartOfISOYear(N, \'Asia/Istanbul\') Code: 43 "Date","2018-12-31" "Date","2018-12-31" ------------------------------------------ -SELECT toStartOfQuarter(N, \'Europe/Moscow\') +SELECT toStartOfQuarter(N, \'Asia/Istanbul\') Code: 43 "Date","2019-07-01" "Date","2019-07-01" ------------------------------------------ -SELECT toStartOfMonth(N, \'Europe/Moscow\') +SELECT toStartOfMonth(N, \'Asia/Istanbul\') Code: 43 "Date","2019-09-01" "Date","2019-09-01" ------------------------------------------ -SELECT toMonday(N, \'Europe/Moscow\') +SELECT toMonday(N, \'Asia/Istanbul\') Code: 43 "Date","2019-09-16" "Date","2019-09-16" ------------------------------------------ -SELECT toStartOfWeek(N, \'Europe/Moscow\') +SELECT toStartOfWeek(N, \'Asia/Istanbul\') Code: 43 Code: 43 Code: 43 ------------------------------------------ -SELECT toStartOfDay(N, \'Europe/Moscow\') -"DateTime('Europe/Moscow')","2019-09-16 00:00:00" -"DateTime('Europe/Moscow')","2019-09-16 00:00:00" -"DateTime('Europe/Moscow')","2019-09-16 00:00:00" +SELECT toStartOfDay(N, \'Asia/Istanbul\') +"DateTime('Asia/Istanbul')","2019-09-16 00:00:00" +"DateTime('Asia/Istanbul')","2019-09-16 00:00:00" +"DateTime('Asia/Istanbul')","2019-09-16 00:00:00" ------------------------------------------ -SELECT toStartOfHour(N, \'Europe/Moscow\') +SELECT toStartOfHour(N, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 19:00:00" -"DateTime('Europe/Moscow')","2019-09-16 19:00:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:00:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:00:00" ------------------------------------------ -SELECT toStartOfMinute(N, \'Europe/Moscow\') +SELECT toStartOfMinute(N, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 19:20:00" -"DateTime('Europe/Moscow')","2019-09-16 19:20:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:20:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:20:00" ------------------------------------------ -SELECT toStartOfFiveMinute(N, \'Europe/Moscow\') +SELECT toStartOfFiveMinute(N, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 19:20:00" -"DateTime('Europe/Moscow')","2019-09-16 19:20:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:20:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:20:00" ------------------------------------------ -SELECT toStartOfTenMinutes(N, \'Europe/Moscow\') +SELECT toStartOfTenMinutes(N, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 19:20:00" -"DateTime('Europe/Moscow')","2019-09-16 19:20:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:20:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:20:00" ------------------------------------------ -SELECT toStartOfFifteenMinutes(N, \'Europe/Moscow\') +SELECT toStartOfFifteenMinutes(N, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 19:15:00" -"DateTime('Europe/Moscow')","2019-09-16 19:15:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:15:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:15:00" ------------------------------------------ -SELECT toStartOfInterval(N, INTERVAL 1 year, \'Europe/Moscow\') +SELECT toStartOfInterval(N, INTERVAL 1 year, \'Asia/Istanbul\') Code: 43 "Date","2019-01-01" "Date","2019-01-01" ------------------------------------------ -SELECT toStartOfInterval(N, INTERVAL 1 month, \'Europe/Moscow\') +SELECT toStartOfInterval(N, INTERVAL 1 month, \'Asia/Istanbul\') Code: 43 "Date","2019-09-01" "Date","2019-09-01" ------------------------------------------ -SELECT toStartOfInterval(N, INTERVAL 1 day, \'Europe/Moscow\') -"DateTime('Europe/Moscow')","2019-09-16 00:00:00" -"DateTime('Europe/Moscow')","2019-09-16 00:00:00" -"DateTime('Europe/Moscow')","2019-09-16 00:00:00" +SELECT toStartOfInterval(N, INTERVAL 1 day, \'Asia/Istanbul\') +"DateTime('Asia/Istanbul')","2019-09-16 00:00:00" +"DateTime('Asia/Istanbul')","2019-09-16 00:00:00" +"DateTime('Asia/Istanbul')","2019-09-16 00:00:00" ------------------------------------------ -SELECT toStartOfInterval(N, INTERVAL 15 minute, \'Europe/Moscow\') +SELECT toStartOfInterval(N, INTERVAL 15 minute, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 19:15:00" -"DateTime('Europe/Moscow')","2019-09-16 19:15:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:15:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:15:00" ------------------------------------------ -SELECT date_trunc(\'year\', N, \'Europe/Moscow\') +SELECT date_trunc(\'year\', N, \'Asia/Istanbul\') Code: 43 "Date","2019-01-01" "Date","2019-01-01" ------------------------------------------ -SELECT date_trunc(\'month\', N, \'Europe/Moscow\') +SELECT date_trunc(\'month\', N, \'Asia/Istanbul\') Code: 43 "Date","2019-09-01" "Date","2019-09-01" ------------------------------------------ -SELECT date_trunc(\'day\', N, \'Europe/Moscow\') -"DateTime('Europe/Moscow')","2019-09-16 00:00:00" -"DateTime('Europe/Moscow')","2019-09-16 00:00:00" -"DateTime('Europe/Moscow')","2019-09-16 00:00:00" +SELECT date_trunc(\'day\', N, \'Asia/Istanbul\') +"DateTime('Asia/Istanbul')","2019-09-16 00:00:00" +"DateTime('Asia/Istanbul')","2019-09-16 00:00:00" +"DateTime('Asia/Istanbul')","2019-09-16 00:00:00" ------------------------------------------ -SELECT date_trunc(\'minute\', N, \'Europe/Moscow\') +SELECT date_trunc(\'minute\', N, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 19:20:00" -"DateTime('Europe/Moscow')","2019-09-16 19:20:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:20:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:20:00" ------------------------------------------ -SELECT toTime(N, \'Europe/Moscow\') +SELECT toTime(N, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","1970-01-02 19:20:11" -"DateTime('Europe/Moscow')","1970-01-02 19:20:11" +"DateTime('Asia/Istanbul')","1970-01-02 19:20:11" +"DateTime('Asia/Istanbul')","1970-01-02 19:20:11" ------------------------------------------ -SELECT toRelativeYearNum(N, \'Europe/Moscow\') +SELECT toRelativeYearNum(N, \'Asia/Istanbul\') "UInt16",2019 "UInt16",2019 "UInt16",2019 ------------------------------------------ -SELECT toRelativeQuarterNum(N, \'Europe/Moscow\') +SELECT toRelativeQuarterNum(N, \'Asia/Istanbul\') "UInt32",8078 "UInt32",8078 "UInt32",8078 ------------------------------------------ -SELECT toRelativeMonthNum(N, \'Europe/Moscow\') +SELECT toRelativeMonthNum(N, \'Asia/Istanbul\') "UInt32",24237 "UInt32",24237 "UInt32",24237 ------------------------------------------ -SELECT toRelativeWeekNum(N, \'Europe/Moscow\') +SELECT toRelativeWeekNum(N, \'Asia/Istanbul\') "UInt32",2594 "UInt32",2594 "UInt32",2594 ------------------------------------------ -SELECT toRelativeDayNum(N, \'Europe/Moscow\') +SELECT toRelativeDayNum(N, \'Asia/Istanbul\') "UInt32",18155 "UInt32",18155 "UInt32",18155 ------------------------------------------ -SELECT toRelativeHourNum(N, \'Europe/Moscow\') +SELECT toRelativeHourNum(N, \'Asia/Istanbul\') "UInt32",435717 "UInt32",435736 "UInt32",435736 ------------------------------------------ -SELECT toRelativeMinuteNum(N, \'Europe/Moscow\') +SELECT toRelativeMinuteNum(N, \'Asia/Istanbul\') "UInt32",26143020 "UInt32",26144180 "UInt32",26144180 ------------------------------------------ -SELECT toRelativeSecondNum(N, \'Europe/Moscow\') +SELECT toRelativeSecondNum(N, \'Asia/Istanbul\') "UInt32",1568581200 "UInt32",1568650811 "UInt32",1568650811 ------------------------------------------ -SELECT toISOYear(N, \'Europe/Moscow\') +SELECT toISOYear(N, \'Asia/Istanbul\') "UInt16",2019 "UInt16",2019 "UInt16",2019 ------------------------------------------ -SELECT toISOWeek(N, \'Europe/Moscow\') +SELECT toISOWeek(N, \'Asia/Istanbul\') "UInt8",38 "UInt8",38 "UInt8",38 ------------------------------------------ -SELECT toWeek(N, \'Europe/Moscow\') +SELECT toWeek(N, \'Asia/Istanbul\') Code: 43 Code: 43 Code: 43 ------------------------------------------ -SELECT toYearWeek(N, \'Europe/Moscow\') +SELECT toYearWeek(N, \'Asia/Istanbul\') Code: 43 Code: 43 Code: 43 ------------------------------------------ -SELECT timeSlot(N, \'Europe/Moscow\') +SELECT timeSlot(N, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 19:00:00" -"DateTime('Europe/Moscow')","2019-09-16 19:00:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:00:00" +"DateTime('Asia/Istanbul')","2019-09-16 19:00:00" ------------------------------------------ -SELECT toYYYYMM(N, \'Europe/Moscow\') +SELECT toYYYYMM(N, \'Asia/Istanbul\') "UInt32",201909 "UInt32",201909 "UInt32",201909 ------------------------------------------ -SELECT toYYYYMMDD(N, \'Europe/Moscow\') +SELECT toYYYYMMDD(N, \'Asia/Istanbul\') "UInt32",20190916 "UInt32",20190916 "UInt32",20190916 ------------------------------------------ -SELECT toYYYYMMDDhhmmss(N, \'Europe/Moscow\') +SELECT toYYYYMMDDhhmmss(N, \'Asia/Istanbul\') "UInt64",20190916000000 "UInt64",20190916192011 "UInt64",20190916192011 ------------------------------------------ -SELECT addYears(N, 1, \'Europe/Moscow\') +SELECT addYears(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2020-09-16 19:20:11" +"DateTime('Asia/Istanbul')","2020-09-16 19:20:11" Code: 43 ------------------------------------------ -SELECT addMonths(N, 1, \'Europe/Moscow\') +SELECT addMonths(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-10-16 19:20:11" +"DateTime('Asia/Istanbul')","2019-10-16 19:20:11" Code: 43 ------------------------------------------ -SELECT addWeeks(N, 1, \'Europe/Moscow\') +SELECT addWeeks(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-23 19:20:11" +"DateTime('Asia/Istanbul')","2019-09-23 19:20:11" Code: 43 ------------------------------------------ -SELECT addDays(N, 1, \'Europe/Moscow\') +SELECT addDays(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-17 19:20:11" +"DateTime('Asia/Istanbul')","2019-09-17 19:20:11" Code: 43 ------------------------------------------ -SELECT addHours(N, 1, \'Europe/Moscow\') +SELECT addHours(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 20:20:11" +"DateTime('Asia/Istanbul')","2019-09-16 20:20:11" Code: 43 ------------------------------------------ -SELECT addMinutes(N, 1, \'Europe/Moscow\') +SELECT addMinutes(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 19:21:11" +"DateTime('Asia/Istanbul')","2019-09-16 19:21:11" Code: 43 ------------------------------------------ -SELECT addSeconds(N, 1, \'Europe/Moscow\') +SELECT addSeconds(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 19:20:12" +"DateTime('Asia/Istanbul')","2019-09-16 19:20:12" Code: 43 ------------------------------------------ -SELECT addQuarters(N, 1, \'Europe/Moscow\') +SELECT addQuarters(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-12-16 19:20:11" +"DateTime('Asia/Istanbul')","2019-12-16 19:20:11" Code: 43 ------------------------------------------ -SELECT subtractYears(N, 1, \'Europe/Moscow\') +SELECT subtractYears(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2018-09-16 19:20:11" +"DateTime('Asia/Istanbul')","2018-09-16 19:20:11" Code: 43 ------------------------------------------ -SELECT subtractMonths(N, 1, \'Europe/Moscow\') +SELECT subtractMonths(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-08-16 19:20:11" +"DateTime('Asia/Istanbul')","2019-08-16 19:20:11" Code: 43 ------------------------------------------ -SELECT subtractWeeks(N, 1, \'Europe/Moscow\') +SELECT subtractWeeks(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-09 19:20:11" +"DateTime('Asia/Istanbul')","2019-09-09 19:20:11" Code: 43 ------------------------------------------ -SELECT subtractDays(N, 1, \'Europe/Moscow\') +SELECT subtractDays(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-15 19:20:11" +"DateTime('Asia/Istanbul')","2019-09-15 19:20:11" Code: 43 ------------------------------------------ -SELECT subtractHours(N, 1, \'Europe/Moscow\') +SELECT subtractHours(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 18:20:11" +"DateTime('Asia/Istanbul')","2019-09-16 18:20:11" Code: 43 ------------------------------------------ -SELECT subtractMinutes(N, 1, \'Europe/Moscow\') +SELECT subtractMinutes(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 19:19:11" +"DateTime('Asia/Istanbul')","2019-09-16 19:19:11" Code: 43 ------------------------------------------ -SELECT subtractSeconds(N, 1, \'Europe/Moscow\') +SELECT subtractSeconds(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-09-16 19:20:10" +"DateTime('Asia/Istanbul')","2019-09-16 19:20:10" Code: 43 ------------------------------------------ -SELECT subtractQuarters(N, 1, \'Europe/Moscow\') +SELECT subtractQuarters(N, 1, \'Asia/Istanbul\') Code: 43 -"DateTime('Europe/Moscow')","2019-06-16 19:20:11" +"DateTime('Asia/Istanbul')","2019-06-16 19:20:11" Code: 43 ------------------------------------------ SELECT CAST(N as DateTime(\'Europe/Minsk\')) @@ -353,7 +353,7 @@ SELECT CAST(N as DateTime64(9, \'Europe/Minsk\')) "DateTime64(9, 'Europe/Minsk')","2019-09-16 19:20:11.000000000" "DateTime64(9, 'Europe/Minsk')","2019-09-16 19:20:11.234000000" ------------------------------------------ -SELECT formatDateTime(N, \'%C %d %D %e %F %H %I %j %m %M %p %R %S %T %u %V %w %y %Y %%\', \'Europe/Moscow\') +SELECT formatDateTime(N, \'%C %d %D %e %F %H %I %j %m %M %p %R %S %T %u %V %w %y %Y %%\', \'Asia/Istanbul\') "String","20 16 09/16/19 16 2019-09-16 00 12 259 09 00 AM 00:00 00 00:00:00 1 38 1 19 2019 %" "String","20 16 09/16/19 16 2019-09-16 19 07 259 09 20 PM 19:20 11 19:20:11 1 38 1 19 2019 %" "String","20 16 09/16/19 16 2019-09-16 19 07 259 09 20 PM 19:20 11 19:20:11 1 38 1 19 2019 %" diff --git a/tests/queries/0_stateless/00927_asof_join_other_types.sh b/tests/queries/0_stateless/00927_asof_join_other_types.sh index 0c17ca2085f..10173a3e43f 100755 --- a/tests/queries/0_stateless/00927_asof_join_other_types.sh +++ b/tests/queries/0_stateless/00927_asof_join_other_types.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -for typename in "UInt32" "UInt64" "Float64" "Float32" "DateTime('Europe/Moscow')" "Decimal32(5)" "Decimal64(5)" "Decimal128(5)" "DateTime64(3, 'Europe/Moscow')" +for typename in "UInt32" "UInt64" "Float64" "Float32" "DateTime('Asia/Istanbul')" "Decimal32(5)" "Decimal64(5)" "Decimal128(5)" "DateTime64(3, 'Asia/Istanbul')" do $CLICKHOUSE_CLIENT -mn <= 5 LIMIT 15); -INSERT INTO bloom_filter_array_types_test SELECT groupArray(number) AS order_key, groupArray(toInt8(number)) AS i8, groupArray(toInt16(number)) AS i16, groupArray(toInt32(number)) AS i32, groupArray(toInt64(number)) AS i64, groupArray(toUInt8(number)) AS u8, groupArray(toUInt16(number)) AS u16, groupArray(toUInt32(number)) AS u32, groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, groupArray(toDate(number, 'Europe/Moscow')) AS date, groupArray(toDateTime(number, 'Europe/Moscow')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers WHERE number >= 10 LIMIT 15); +CREATE TABLE bloom_filter_array_types_test (order_key Array(UInt64), i8 Array(Int8), i16 Array(Int16), i32 Array(Int32), i64 Array(Int64), u8 Array(UInt8), u16 Array(UInt16), u32 Array(UInt32), u64 Array(UInt64), f32 Array(Float32), f64 Array(Float64), date Array(Date), date_time Array(DateTime('Asia/Istanbul')), str Array(String), fixed_string Array(FixedString(5)), INDEX idx (i8, i16, i32, i64, u8, u16, u32, u64, f32, f64, date, date_time, str, fixed_string) TYPE bloom_filter GRANULARITY 1) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 6; +INSERT INTO bloom_filter_array_types_test SELECT groupArray(number) AS order_key, groupArray(toInt8(number)) AS i8, groupArray(toInt16(number)) AS i16, groupArray(toInt32(number)) AS i32, groupArray(toInt64(number)) AS i64, groupArray(toUInt8(number)) AS u8, groupArray(toUInt16(number)) AS u16, groupArray(toUInt32(number)) AS u32, groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, groupArray(toDate(number, 'Asia/Istanbul')) AS date, groupArray(toDateTime(number, 'Asia/Istanbul')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers LIMIT 15); +INSERT INTO bloom_filter_array_types_test SELECT groupArray(number) AS order_key, groupArray(toInt8(number)) AS i8, groupArray(toInt16(number)) AS i16, groupArray(toInt32(number)) AS i32, groupArray(toInt64(number)) AS i64, groupArray(toUInt8(number)) AS u8, groupArray(toUInt16(number)) AS u16, groupArray(toUInt32(number)) AS u32, groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, groupArray(toDate(number, 'Asia/Istanbul')) AS date, groupArray(toDateTime(number, 'Asia/Istanbul')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers WHERE number >= 5 LIMIT 15); +INSERT INTO bloom_filter_array_types_test SELECT groupArray(number) AS order_key, groupArray(toInt8(number)) AS i8, groupArray(toInt16(number)) AS i16, groupArray(toInt32(number)) AS i32, groupArray(toInt64(number)) AS i64, groupArray(toUInt8(number)) AS u8, groupArray(toUInt16(number)) AS u16, groupArray(toUInt32(number)) AS u32, groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, groupArray(toDate(number, 'Asia/Istanbul')) AS date, groupArray(toDateTime(number, 'Asia/Istanbul')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers WHERE number >= 10 LIMIT 15); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(i8, 1); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(i16, 1); @@ -68,7 +68,7 @@ SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(u64, 1); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(f32, 1); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(f64, 1); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(date, toDate('1970-01-02')); -SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(date_time, toDateTime('1970-01-01 03:00:01', 'Europe/Moscow')); +SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(date_time, toDateTime('1970-01-01 03:00:01', 'Asia/Istanbul')); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(str, '1'); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(fixed_string, toFixedString('1', 5)); @@ -83,7 +83,7 @@ SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(u64, 5); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(f32, 5); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(f64, 5); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(date, toDate('1970-01-06')); -SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(date_time, toDateTime('1970-01-01 03:00:05', 'Europe/Moscow')); +SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(date_time, toDateTime('1970-01-01 03:00:05', 'Asia/Istanbul')); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(str, '5'); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(fixed_string, toFixedString('5', 5)); @@ -98,7 +98,7 @@ SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(u64, 10); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(f32, 10); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(f64, 10); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(date, toDate('1970-01-11')); -SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(date_time, toDateTime('1970-01-01 03:00:10', 'Europe/Moscow')); +SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(date_time, toDateTime('1970-01-01 03:00:10', 'Asia/Istanbul')); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(str, '10'); SELECT COUNT() FROM bloom_filter_array_types_test WHERE has(fixed_string, toFixedString('10', 5)); @@ -106,8 +106,8 @@ DROP TABLE IF EXISTS bloom_filter_array_types_test; DROP TABLE IF EXISTS bloom_filter_null_types_test; -CREATE TABLE bloom_filter_null_types_test (order_key UInt64, i8 Nullable(Int8), i16 Nullable(Int16), i32 Nullable(Int32), i64 Nullable(Int64), u8 Nullable(UInt8), u16 Nullable(UInt16), u32 Nullable(UInt32), u64 Nullable(UInt64), f32 Nullable(Float32), f64 Nullable(Float64), date Nullable(Date), date_time Nullable(DateTime('Europe/Moscow')), str Nullable(String), fixed_string Nullable(FixedString(5)), INDEX idx (i8, i16, i32, i64, u8, u16, u32, u64, f32, f64, date, date_time, str, fixed_string) TYPE bloom_filter GRANULARITY 1) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 6; -INSERT INTO bloom_filter_null_types_test SELECT number AS order_key, toInt8(number) AS i8, toInt16(number) AS i16, toInt32(number) AS i32, toInt64(number) AS i64, toUInt8(number) AS u8, toUInt16(number) AS u16, toUInt32(number) AS u32, toUInt64(number) AS u64, toFloat32(number) AS f32, toFloat64(number) AS f64, toDate(number, 'Europe/Moscow') AS date, toDateTime(number, 'Europe/Moscow') AS date_time, toString(number) AS str, toFixedString(toString(number), 5) AS fixed_string FROM system.numbers LIMIT 100; +CREATE TABLE bloom_filter_null_types_test (order_key UInt64, i8 Nullable(Int8), i16 Nullable(Int16), i32 Nullable(Int32), i64 Nullable(Int64), u8 Nullable(UInt8), u16 Nullable(UInt16), u32 Nullable(UInt32), u64 Nullable(UInt64), f32 Nullable(Float32), f64 Nullable(Float64), date Nullable(Date), date_time Nullable(DateTime('Asia/Istanbul')), str Nullable(String), fixed_string Nullable(FixedString(5)), INDEX idx (i8, i16, i32, i64, u8, u16, u32, u64, f32, f64, date, date_time, str, fixed_string) TYPE bloom_filter GRANULARITY 1) ENGINE = MergeTree() ORDER BY order_key SETTINGS index_granularity = 6; +INSERT INTO bloom_filter_null_types_test SELECT number AS order_key, toInt8(number) AS i8, toInt16(number) AS i16, toInt32(number) AS i32, toInt64(number) AS i64, toUInt8(number) AS u8, toUInt16(number) AS u16, toUInt32(number) AS u32, toUInt64(number) AS u64, toFloat32(number) AS f32, toFloat64(number) AS f64, toDate(number, 'Asia/Istanbul') AS date, toDateTime(number, 'Asia/Istanbul') AS date_time, toString(number) AS str, toFixedString(toString(number), 5) AS fixed_string FROM system.numbers LIMIT 100; INSERT INTO bloom_filter_null_types_test SELECT 0 AS order_key, NULL AS i8, NULL AS i16, NULL AS i32, NULL AS i64, NULL AS u8, NULL AS u16, NULL AS u32, NULL AS u64, NULL AS f32, NULL AS f64, NULL AS date, NULL AS date_time, NULL AS str, NULL AS fixed_string; SELECT COUNT() FROM bloom_filter_null_types_test WHERE i8 = 1 SETTINGS max_rows_to_read = 6; @@ -121,7 +121,7 @@ SELECT COUNT() FROM bloom_filter_null_types_test WHERE u64 = 1 SETTINGS max_rows SELECT COUNT() FROM bloom_filter_null_types_test WHERE f32 = 1 SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_null_types_test WHERE f64 = 1 SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_null_types_test WHERE date = '1970-01-02' SETTINGS max_rows_to_read = 6; -SELECT COUNT() FROM bloom_filter_null_types_test WHERE date_time = toDateTime('1970-01-01 03:00:01', 'Europe/Moscow') SETTINGS max_rows_to_read = 6; +SELECT COUNT() FROM bloom_filter_null_types_test WHERE date_time = toDateTime('1970-01-01 03:00:01', 'Asia/Istanbul') SETTINGS max_rows_to_read = 6; SELECT COUNT() FROM bloom_filter_null_types_test WHERE str = '1' SETTINGS max_rows_to_read = 12; SELECT COUNT() FROM bloom_filter_null_types_test WHERE fixed_string = toFixedString('1', 5) SETTINGS max_rows_to_read = 12; @@ -177,7 +177,7 @@ CREATE TABLE bloom_filter_array_lc_null_types_test ( f64 Array(LowCardinality(Nullable(Float64))), date Array(LowCardinality(Nullable(Date))), - date_time Array(LowCardinality(Nullable(DateTime('Europe/Moscow')))), + date_time Array(LowCardinality(Nullable(DateTime('Asia/Istanbul')))), str Array(LowCardinality(Nullable(String))), fixed_string Array(LowCardinality(Nullable(FixedString(5)))), @@ -197,16 +197,16 @@ SELECT groupArray(number) AS order_key, groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, - groupArray(toDate(number, 'Europe/Moscow')) AS date, - groupArray(toDateTime(number, 'Europe/Moscow')) AS date_time, + groupArray(toDate(number, 'Asia/Istanbul')) AS date, + groupArray(toDateTime(number, 'Asia/Istanbul')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers LIMIT 15); -INSERT INTO bloom_filter_array_lc_null_types_test SELECT groupArray(number) AS order_key, groupArray(toInt8(number)) AS i8, groupArray(toInt16(number)) AS i16, groupArray(toInt32(number)) AS i32, groupArray(toInt64(number)) AS i64, groupArray(toUInt8(number)) AS u8, groupArray(toUInt16(number)) AS u16, groupArray(toUInt32(number)) AS u32, groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, groupArray(toDate(number, 'Europe/Moscow')) AS date, groupArray(toDateTime(number, 'Europe/Moscow')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers WHERE number >= 5 LIMIT 15); -INSERT INTO bloom_filter_array_lc_null_types_test SELECT groupArray(number) AS order_key, groupArray(toInt8(number)) AS i8, groupArray(toInt16(number)) AS i16, groupArray(toInt32(number)) AS i32, groupArray(toInt64(number)) AS i64, groupArray(toUInt8(number)) AS u8, groupArray(toUInt16(number)) AS u16, groupArray(toUInt32(number)) AS u32, groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, groupArray(toDate(number, 'Europe/Moscow')) AS date, groupArray(toDateTime(number, 'Europe/Moscow')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers WHERE number >= 10 LIMIT 15); +INSERT INTO bloom_filter_array_lc_null_types_test SELECT groupArray(number) AS order_key, groupArray(toInt8(number)) AS i8, groupArray(toInt16(number)) AS i16, groupArray(toInt32(number)) AS i32, groupArray(toInt64(number)) AS i64, groupArray(toUInt8(number)) AS u8, groupArray(toUInt16(number)) AS u16, groupArray(toUInt32(number)) AS u32, groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, groupArray(toDate(number, 'Asia/Istanbul')) AS date, groupArray(toDateTime(number, 'Asia/Istanbul')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers WHERE number >= 5 LIMIT 15); +INSERT INTO bloom_filter_array_lc_null_types_test SELECT groupArray(number) AS order_key, groupArray(toInt8(number)) AS i8, groupArray(toInt16(number)) AS i16, groupArray(toInt32(number)) AS i32, groupArray(toInt64(number)) AS i64, groupArray(toUInt8(number)) AS u8, groupArray(toUInt16(number)) AS u16, groupArray(toUInt32(number)) AS u32, groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, groupArray(toDate(number, 'Asia/Istanbul')) AS date, groupArray(toDateTime(number, 'Asia/Istanbul')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers WHERE number >= 10 LIMIT 15); INSERT INTO bloom_filter_array_lc_null_types_test SELECT n AS order_key, n AS i8, n AS i16, n AS i32, n AS i64, n AS u8, n AS u16, n AS u32, n AS u64, n AS f32, n AS f64, n AS date, n AS date_time, n AS str, n AS fixed_string FROM (SELECT [NULL] AS n); -INSERT INTO bloom_filter_array_lc_null_types_test SELECT [NULL, n] AS order_key, [NULL, toInt8(n)] AS i8, [NULL, toInt16(n)] AS i16, [NULL, toInt32(n)] AS i32, [NULL, toInt64(n)] AS i64, [NULL, toUInt8(n)] AS u8, [NULL, toUInt16(n)] AS u16, [NULL, toUInt32(n)] AS u32, [NULL, toUInt64(n)] AS u64, [NULL, toFloat32(n)] AS f32, [NULL, toFloat64(n)] AS f64, [NULL, toDate(n, 'Europe/Moscow')] AS date, [NULL, toDateTime(n, 'Europe/Moscow')] AS date_time, [NULL, toString(n)] AS str, [NULL, toFixedString(toString(n), 5)] AS fixed_string FROM (SELECT 100 as n); +INSERT INTO bloom_filter_array_lc_null_types_test SELECT [NULL, n] AS order_key, [NULL, toInt8(n)] AS i8, [NULL, toInt16(n)] AS i16, [NULL, toInt32(n)] AS i32, [NULL, toInt64(n)] AS i64, [NULL, toUInt8(n)] AS u8, [NULL, toUInt16(n)] AS u16, [NULL, toUInt32(n)] AS u32, [NULL, toUInt64(n)] AS u64, [NULL, toFloat32(n)] AS f32, [NULL, toFloat64(n)] AS f64, [NULL, toDate(n, 'Asia/Istanbul')] AS date, [NULL, toDateTime(n, 'Asia/Istanbul')] AS date_time, [NULL, toString(n)] AS str, [NULL, toFixedString(toString(n), 5)] AS fixed_string FROM (SELECT 100 as n); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(i8, 1); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(i16, 1); @@ -219,7 +219,7 @@ SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(u64, 1); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(f32, 1); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(f64, 1); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(date, toDate('1970-01-02')); -SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(date_time, toDateTime('1970-01-01 03:00:01', 'Europe/Moscow')); +SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(date_time, toDateTime('1970-01-01 03:00:01', 'Asia/Istanbul')); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(str, '1'); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(fixed_string, toFixedString('1', 5)); @@ -234,7 +234,7 @@ SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(u64, 5); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(f32, 5); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(f64, 5); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(date, toDate('1970-01-06')); -SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(date_time, toDateTime('1970-01-01 03:00:05', 'Europe/Moscow')); +SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(date_time, toDateTime('1970-01-01 03:00:05', 'Asia/Istanbul')); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(str, '5'); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(fixed_string, toFixedString('5', 5)); @@ -249,7 +249,7 @@ SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(u64, 10); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(f32, 10); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(f64, 10); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(date, toDate('1970-01-11')); -SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(date_time, toDateTime('1970-01-01 03:00:10', 'Europe/Moscow')); +SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(date_time, toDateTime('1970-01-01 03:00:10', 'Asia/Istanbul')); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(str, '10'); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(fixed_string, toFixedString('10', 5)); @@ -279,7 +279,7 @@ SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(u64, 100); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(f32, 100); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(f64, 100); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(date, toDate('1970-04-11')); -SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(date_time, toDateTime('1970-01-01 03:01:40', 'Europe/Moscow')); +SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(date_time, toDateTime('1970-01-01 03:01:40', 'Asia/Istanbul')); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(str, '100'); SELECT COUNT() FROM bloom_filter_array_lc_null_types_test WHERE has(fixed_string, toFixedString('100', 5)); diff --git a/tests/queries/0_stateless/01077_mutations_index_consistency.sh b/tests/queries/0_stateless/01077_mutations_index_consistency.sh index 31086ed6784..c41eab62ecb 100755 --- a/tests/queries/0_stateless/01077_mutations_index_consistency.sh +++ b/tests/queries/0_stateless/01077_mutations_index_consistency.sh @@ -7,9 +7,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS movement" -$CLICKHOUSE_CLIENT -n --query "CREATE TABLE movement (date DateTime('Europe/Moscow')) Engine = MergeTree ORDER BY (toStartOfHour(date));" +$CLICKHOUSE_CLIENT -n --query "CREATE TABLE movement (date DateTime('Asia/Istanbul')) Engine = MergeTree ORDER BY (toStartOfHour(date));" -$CLICKHOUSE_CLIENT --query "insert into movement select toDateTime('2020-01-22 00:00:00', 'Europe/Moscow') + number%(23*3600) from numbers(1000000);" +$CLICKHOUSE_CLIENT --query "insert into movement select toDateTime('2020-01-22 00:00:00', 'Asia/Istanbul') + number%(23*3600) from numbers(1000000);" $CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE movement FINAL" @@ -18,20 +18,20 @@ SELECT count(), toStartOfHour(date) AS Hour FROM movement -WHERE (date >= toDateTime('2020-01-22T10:00:00', 'Europe/Moscow')) AND (date <= toDateTime('2020-01-22T23:00:00', 'Europe/Moscow')) +WHERE (date >= toDateTime('2020-01-22T10:00:00', 'Asia/Istanbul')) AND (date <= toDateTime('2020-01-22T23:00:00', 'Asia/Istanbul')) GROUP BY Hour ORDER BY Hour DESC " | grep "16:00:00" | cut -f1 -$CLICKHOUSE_CLIENT --query "alter table movement delete where date >= toDateTime('2020-01-22T16:00:00', 'Europe/Moscow') and date < toDateTime('2020-01-22T17:00:00', 'Europe/Moscow') SETTINGS mutations_sync = 2" +$CLICKHOUSE_CLIENT --query "alter table movement delete where date >= toDateTime('2020-01-22T16:00:00', 'Asia/Istanbul') and date < toDateTime('2020-01-22T17:00:00', 'Asia/Istanbul') SETTINGS mutations_sync = 2" $CLICKHOUSE_CLIENT -n --query " SELECT count(), toStartOfHour(date) AS Hour FROM movement -WHERE (date >= toDateTime('2020-01-22T10:00:00', 'Europe/Moscow')) AND (date <= toDateTime('2020-01-22T23:00:00', 'Europe/Moscow')) +WHERE (date >= toDateTime('2020-01-22T10:00:00', 'Asia/Istanbul')) AND (date <= toDateTime('2020-01-22T23:00:00', 'Asia/Istanbul')) GROUP BY Hour ORDER BY Hour DESC " | grep "16:00:00" | wc -l @@ -42,7 +42,7 @@ SELECT count(), toStartOfHour(date) AS Hour FROM movement -WHERE (date >= toDateTime('2020-01-22T10:00:00', 'Europe/Moscow')) AND (date <= toDateTime('2020-01-22T23:00:00', 'Europe/Moscow')) +WHERE (date >= toDateTime('2020-01-22T10:00:00', 'Asia/Istanbul')) AND (date <= toDateTime('2020-01-22T23:00:00', 'Asia/Istanbul')) GROUP BY Hour ORDER BY Hour DESC " | grep "22:00:00" | cut -f1 diff --git a/tests/queries/0_stateless/01087_storage_generate.sql b/tests/queries/0_stateless/01087_storage_generate.sql index a16ad55832c..7df9f3931d0 100644 --- a/tests/queries/0_stateless/01087_storage_generate.sql +++ b/tests/queries/0_stateless/01087_storage_generate.sql @@ -7,7 +7,7 @@ DROP TABLE IF EXISTS test_table; SELECT '-'; DROP TABLE IF EXISTS test_table_2; -CREATE TABLE test_table_2(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3, 'Europe/Moscow'), UUID)) ENGINE=GenerateRandom(10, 5, 3); +CREATE TABLE test_table_2(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3, 'Asia/Istanbul'), UUID)) ENGINE=GenerateRandom(10, 5, 3); SELECT * FROM test_table_2 LIMIT 100; diff --git a/tests/queries/0_stateless/01087_table_function_generate.reference b/tests/queries/0_stateless/01087_table_function_generate.reference index ead4e97403b..ef7eac41ca2 100644 --- a/tests/queries/0_stateless/01087_table_function_generate.reference +++ b/tests/queries/0_stateless/01087_table_function_generate.reference @@ -46,7 +46,7 @@ h \N o - -Date DateTime(\'Europe/Moscow\') DateTime(\'Europe/Moscow\') +Date DateTime(\'Asia/Istanbul\') DateTime(\'Asia/Istanbul\') 2113-06-12 2050-12-17 02:46:35 2096-02-16 22:18:22 2141-08-09 2013-10-17 23:35:26 1976-01-24 12:52:48 2039-08-16 1974-11-17 23:22:46 1980-03-04 21:02:50 @@ -58,7 +58,7 @@ Date DateTime(\'Europe/Moscow\') DateTime(\'Europe/Moscow\') 2008-03-16 2047-05-16 23:28:36 2103-02-11 16:44:39 2000-07-07 2105-07-19 19:29:06 1980-01-02 05:18:22 - -DateTime64(3, \'Europe/Moscow\') DateTime64(6, \'Europe/Moscow\') DateTime64(6, \'Europe/Moscow\') +DateTime64(3, \'Asia/Istanbul\') DateTime64(6, \'Asia/Istanbul\') DateTime64(6, \'Asia/Istanbul\') 1978-06-07 23:50:57.320 2013-08-28 10:21:54.010758 1991-08-25 16:23:26.140215 1978-08-25 17:07:25.427 2034-05-02 20:49:42.148578 2015-08-26 15:26:31.783160 2037-04-04 10:50:56.898 2055-05-28 11:12:48.819271 2068-12-26 09:58:49.635722 diff --git a/tests/queries/0_stateless/01087_table_function_generate.sql b/tests/queries/0_stateless/01087_table_function_generate.sql index ef4311649f7..512121b7ecc 100644 --- a/tests/queries/0_stateless/01087_table_function_generate.sql +++ b/tests/queries/0_stateless/01087_table_function_generate.sql @@ -42,20 +42,20 @@ LIMIT 10; SELECT '-'; SELECT toTypeName(d), toTypeName(dt), toTypeName(dtm) -FROM generateRandom('d Date, dt DateTime(\'Europe/Moscow\'), dtm DateTime(\'Europe/Moscow\')') +FROM generateRandom('d Date, dt DateTime(\'Asia/Istanbul\'), dtm DateTime(\'Asia/Istanbul\')') LIMIT 1; SELECT d, dt, dtm -FROM generateRandom('d Date, dt DateTime(\'Europe/Moscow\'), dtm DateTime(\'Europe/Moscow\')', 1, 10, 10) +FROM generateRandom('d Date, dt DateTime(\'Asia/Istanbul\'), dtm DateTime(\'Asia/Istanbul\')', 1, 10, 10) LIMIT 10; SELECT '-'; SELECT toTypeName(dt64), toTypeName(dts64), toTypeName(dtms64) -FROM generateRandom('dt64 DateTime64(3, \'Europe/Moscow\'), dts64 DateTime64(6, \'Europe/Moscow\'), dtms64 DateTime64(6 ,\'Europe/Moscow\')') +FROM generateRandom('dt64 DateTime64(3, \'Asia/Istanbul\'), dts64 DateTime64(6, \'Asia/Istanbul\'), dtms64 DateTime64(6 ,\'Asia/Istanbul\')') LIMIT 1; SELECT dt64, dts64, dtms64 -FROM generateRandom('dt64 DateTime64(3, \'Europe/Moscow\'), dts64 DateTime64(6, \'Europe/Moscow\'), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 1, 10, 10) +FROM generateRandom('dt64 DateTime64(3, \'Asia/Istanbul\'), dts64 DateTime64(6, \'Asia/Istanbul\'), dtms64 DateTime64(6 ,\'Asia/Istanbul\')', 1, 10, 10) LIMIT 10; SELECT toTypeName(d32) @@ -176,8 +176,8 @@ FROM generateRandom('i String', 1, 10, 10) LIMIT 10; SELECT '-'; DROP TABLE IF EXISTS test_table; -CREATE TABLE test_table(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3, 'Europe/Moscow'), UUID)) ENGINE=Memory; -INSERT INTO test_table SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3, \'Europe/Moscow\'), UUID)', 1, 10, 2) +CREATE TABLE test_table(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3, 'Asia/Istanbul'), UUID)) ENGINE=Memory; +INSERT INTO test_table SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3, \'Asia/Istanbul\'), UUID)', 1, 10, 2) LIMIT 10; SELECT * FROM test_table ORDER BY a, d, c; @@ -187,8 +187,8 @@ DROP TABLE IF EXISTS test_table; SELECT '-'; DROP TABLE IF EXISTS test_table_2; -CREATE TABLE test_table_2(a Array(Int8), b UInt32, c Nullable(String), d Decimal32(4), e Nullable(Enum16('h' = 1, 'w' = 5 , 'o' = -200)), f Float64, g Tuple(Date, DateTime('Europe/Moscow'), DateTime64(3, 'Europe/Moscow'), UUID), h FixedString(2)) ENGINE=Memory; -INSERT INTO test_table_2 SELECT * FROM generateRandom('a Array(Int8), b UInt32, c Nullable(String), d Decimal32(4), e Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)), f Float64, g Tuple(Date, DateTime(\'Europe/Moscow\'), DateTime64(3, \'Europe/Moscow\'), UUID), h FixedString(2)', 10, 5, 3) +CREATE TABLE test_table_2(a Array(Int8), b UInt32, c Nullable(String), d Decimal32(4), e Nullable(Enum16('h' = 1, 'w' = 5 , 'o' = -200)), f Float64, g Tuple(Date, DateTime('Asia/Istanbul'), DateTime64(3, 'Asia/Istanbul'), UUID), h FixedString(2)) ENGINE=Memory; +INSERT INTO test_table_2 SELECT * FROM generateRandom('a Array(Int8), b UInt32, c Nullable(String), d Decimal32(4), e Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)), f Float64, g Tuple(Date, DateTime(\'Asia/Istanbul\'), DateTime64(3, \'Asia/Istanbul\'), UUID), h FixedString(2)', 10, 5, 3) LIMIT 10; SELECT a, b, c, d, e, f, g, hex(h) FROM test_table_2 ORDER BY a, b, c, d, e, f, g, h; diff --git a/tests/queries/0_stateless/01098_msgpack_format.sh b/tests/queries/0_stateless/01098_msgpack_format.sh index aa982c5478d..24638f33324 100755 --- a/tests/queries/0_stateless/01098_msgpack_format.sh +++ b/tests/queries/0_stateless/01098_msgpack_format.sh @@ -11,7 +11,7 @@ USER_FILES_PATH=$(clickhouse-client --query "select _path,_file from file('nonex $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS msgpack"; -$CLICKHOUSE_CLIENT --query="CREATE TABLE msgpack (uint8 UInt8, uint16 UInt16, uint32 UInt32, uint64 UInt64, int8 Int8, int16 Int16, int32 Int32, int64 Int64, float Float32, double Float64, string String, date Date, datetime DateTime('Europe/Moscow'), datetime64 DateTime64(3, 'Europe/Moscow'), array Array(UInt32)) ENGINE = Memory"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE msgpack (uint8 UInt8, uint16 UInt16, uint32 UInt32, uint64 UInt64, int8 Int8, int16 Int16, int32 Int32, int64 Int64, float Float32, double Float64, string String, date Date, datetime DateTime('Asia/Istanbul'), datetime64 DateTime64(3, 'Asia/Istanbul'), array Array(UInt32)) ENGINE = Memory"; $CLICKHOUSE_CLIENT --query="INSERT INTO msgpack VALUES (255, 65535, 4294967295, 100000000000, -128, -32768, -2147483648, -100000000000, 2.02, 10000.0000001, 'String', 18980, 1639872000, 1639872000000, [1,2,3,4,5]), (4, 1234, 3244467295, 500000000000, -1, -256, -14741221, -7000000000, 100.1, 14321.032141201, 'Another string', 20000, 1839882000, 1639872891123, [5,4,3,2,1]), (42, 42, 42, 42, 42, 42, 42, 42, 42.42, 42.42, '42', 42, 42, 42, [42])"; diff --git a/tests/queries/0_stateless/01186_conversion_to_nullable.sql b/tests/queries/0_stateless/01186_conversion_to_nullable.sql index 828d3cac05b..52f7ee91f52 100644 --- a/tests/queries/0_stateless/01186_conversion_to_nullable.sql +++ b/tests/queries/0_stateless/01186_conversion_to_nullable.sql @@ -2,9 +2,9 @@ select toUInt8(x) from values('x Nullable(String)', '42', NULL, '0', '', '256'); select toInt64(x) from values('x Nullable(String)', '42', NULL, '0', '', '256'); select toDate(x) from values('x Nullable(String)', '2020-12-24', NULL, '0000-00-00', '', '9999-01-01'); -select toDateTime(x, 'Europe/Moscow') from values('x Nullable(String)', '2020-12-24 01:02:03', NULL, '0000-00-00 00:00:00', ''); -select toDateTime64(x, 2, 'Europe/Moscow') from values('x Nullable(String)', '2020-12-24 01:02:03', NULL, '0000-00-00 00:00:00', ''); -select toUnixTimestamp(x, 'Europe/Moscow') from values ('x Nullable(String)', '2000-01-01 13:12:12', NULL, ''); +select toDateTime(x, 'Asia/Istanbul') from values('x Nullable(String)', '2020-12-24 01:02:03', NULL, '0000-00-00 00:00:00', ''); +select toDateTime64(x, 2, 'Asia/Istanbul') from values('x Nullable(String)', '2020-12-24 01:02:03', NULL, '0000-00-00 00:00:00', ''); +select toUnixTimestamp(x, 'Asia/Istanbul') from values ('x Nullable(String)', '2000-01-01 13:12:12', NULL, ''); select toDecimal32(x, 2) from values ('x Nullable(String)', '42', NULL, '3.14159'); select toDecimal64(x, 8) from values ('x Nullable(String)', '42', NULL, '3.14159'); diff --git a/tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper_long.sql b/tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper_long.sql index a831fd18bfe..c5c1f2ebfd3 100644 --- a/tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper_long.sql +++ b/tests/queries/0_stateless/01213_alter_rename_with_default_zookeeper_long.sql @@ -14,7 +14,7 @@ ENGINE = MergeTree() PARTITION BY date ORDER BY key; -INSERT INTO table_rename_with_default (date, key, value1) SELECT toDateTime(toDate('2019-10-01') + number % 3, 'Europe/Moscow'), number, toString(number) from numbers(9); +INSERT INTO table_rename_with_default (date, key, value1) SELECT toDateTime(toDate('2019-10-01') + number % 3, 'Asia/Istanbul'), number, toString(number) from numbers(9); SELECT * FROM table_rename_with_default WHERE key = 1 FORMAT TSVWithNames; @@ -44,7 +44,7 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{database}/test_01213/table_rename_wit ORDER BY tuple() TTL date2 + INTERVAL 500 MONTH; -INSERT INTO table_rename_with_ttl SELECT toDateTime(toDate('2019-10-01') + number % 3, 'Europe/Moscow'), toDateTime(toDate('2018-10-01') + number % 3, 'Europe/Moscow'), toString(number), toString(number) from numbers(9); +INSERT INTO table_rename_with_ttl SELECT toDateTime(toDate('2019-10-01') + number % 3, 'Asia/Istanbul'), toDateTime(toDate('2018-10-01') + number % 3, 'Asia/Istanbul'), toString(number), toString(number) from numbers(9); SELECT * FROM table_rename_with_ttl WHERE value1 = '1' FORMAT TSVWithNames; diff --git a/tests/queries/0_stateless/01269_toStartOfSecond.sql b/tests/queries/0_stateless/01269_toStartOfSecond.sql index b74eaabf351..641da4a15a9 100644 --- a/tests/queries/0_stateless/01269_toStartOfSecond.sql +++ b/tests/queries/0_stateless/01269_toStartOfSecond.sql @@ -4,7 +4,7 @@ SELECT toStartOfSecond(now()); -- {serverError 43} SELECT toStartOfSecond(); -- {serverError 42} SELECT toStartOfSecond(now64(), 123); -- {serverError 43} -WITH toDateTime64('2019-09-16 19:20:11', 3, 'Europe/Moscow') AS dt64 SELECT toStartOfSecond(dt64, 'UTC') AS res, toTypeName(res); +WITH toDateTime64('2019-09-16 19:20:11', 3, 'Asia/Istanbul') AS dt64 SELECT toStartOfSecond(dt64, 'UTC') AS res, toTypeName(res); WITH toDateTime64('2019-09-16 19:20:11', 0, 'UTC') AS dt64 SELECT toStartOfSecond(dt64) AS res, toTypeName(res); WITH toDateTime64('2019-09-16 19:20:11.123', 3, 'UTC') AS dt64 SELECT toStartOfSecond(dt64) AS res, toTypeName(res); WITH toDateTime64('2019-09-16 19:20:11.123', 9, 'UTC') AS dt64 SELECT toStartOfSecond(dt64) AS res, toTypeName(res); diff --git a/tests/queries/0_stateless/01273_arrow_load.sh b/tests/queries/0_stateless/01273_arrow_load.sh index 2e213ce3a79..fa70255821a 100755 --- a/tests/queries/0_stateless/01273_arrow_load.sh +++ b/tests/queries/0_stateless/01273_arrow_load.sh @@ -12,7 +12,7 @@ CB_DIR=$(dirname "$CLICKHOUSE_CLIENT_BINARY") DATA_FILE=$CUR_DIR/data_arrow/test.arrow ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_load" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_load (bool UInt8, int8 Int8, int16 Int16, int32 Int32, int64 Int64, uint8 UInt8, uint16 UInt16, uint32 UInt32, uint64 UInt64, halffloat Float32, float Float32, double Float64, string String, date32 Date, date64 DateTime('Europe/Moscow'), timestamp DateTime('Europe/Moscow')) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_load (bool UInt8, int8 Int8, int16 Int16, int32 Int32, int64 Int64, uint8 UInt8, uint16 UInt16, uint32 UInt32, uint64 UInt64, halffloat Float32, float Float32, double Float64, string String, date32 Date, date64 DateTime('Asia/Istanbul'), timestamp DateTime('Asia/Istanbul')) ENGINE = Memory" cat "$DATA_FILE" | ${CLICKHOUSE_CLIENT} -q "insert into arrow_load format Arrow" ${CLICKHOUSE_CLIENT} --query="select * from arrow_load" diff --git a/tests/queries/0_stateless/01277_toUnixTimestamp64.sql b/tests/queries/0_stateless/01277_toUnixTimestamp64.sql index eb3e8c612ed..42de53beb66 100644 --- a/tests/queries/0_stateless/01277_toUnixTimestamp64.sql +++ b/tests/queries/0_stateless/01277_toUnixTimestamp64.sql @@ -12,22 +12,22 @@ SELECT toUnixTimestamp64Micro('abc', 123); -- {serverError 42} SELECT toUnixTimestamp64Nano('abc', 123); -- {serverError 42} SELECT 'const column'; -WITH toDateTime64('2019-09-16 19:20:12.345678910', 3, 'Europe/Moscow') AS dt64 +WITH toDateTime64('2019-09-16 19:20:12.345678910', 3, 'Asia/Istanbul') AS dt64 SELECT dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); -WITH toDateTime64('2019-09-16 19:20:12.345678910', 6, 'Europe/Moscow') AS dt64 +WITH toDateTime64('2019-09-16 19:20:12.345678910', 6, 'Asia/Istanbul') AS dt64 SELECT dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); -WITH toDateTime64('2019-09-16 19:20:12.345678910', 9, 'Europe/Moscow') AS dt64 +WITH toDateTime64('2019-09-16 19:20:12.345678910', 9, 'Asia/Istanbul') AS dt64 SELECT dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); SELECT 'non-const column'; -WITH toDateTime64('2019-09-16 19:20:12.345678910', 3, 'Europe/Moscow') AS x +WITH toDateTime64('2019-09-16 19:20:12.345678910', 3, 'Asia/Istanbul') AS x SELECT materialize(x) as dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); -WITH toDateTime64('2019-09-16 19:20:12.345678910', 6, 'Europe/Moscow') AS x +WITH toDateTime64('2019-09-16 19:20:12.345678910', 6, 'Asia/Istanbul') AS x SELECT materialize(x) as dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); -WITH toDateTime64('2019-09-16 19:20:12.345678910', 9, 'Europe/Moscow') AS x +WITH toDateTime64('2019-09-16 19:20:12.345678910', 9, 'Asia/Istanbul') AS x SELECT materialize(x) as dt64, toUnixTimestamp64Milli(dt64), toUnixTimestamp64Micro(dt64), toUnixTimestamp64Nano(dt64); diff --git a/tests/queries/0_stateless/01280_min_map_max_map.sql b/tests/queries/0_stateless/01280_min_map_max_map.sql index 96fdfc61929..9bc8c320d93 100644 --- a/tests/queries/0_stateless/01280_min_map_max_map.sql +++ b/tests/queries/0_stateless/01280_min_map_max_map.sql @@ -15,7 +15,7 @@ select minMap(val, cnt) from values ('val Array(FixedString(1)), cnt Array(Fixed select minMap(val, cnt) from values ('val Array(UInt64), cnt Array(UInt64)', ([1], [1]), ([1], [2])); select minMap(val, cnt) from values ('val Array(Float64), cnt Array(Int8)', ([1], [1]), ([1], [2])); select minMap(val, cnt) from values ('val Array(Date), cnt Array(Int16)', ([1], [1]), ([1], [2])); -select minMap(val, cnt) from values ('val Array(DateTime(\'Europe/Moscow\')), cnt Array(Int32)', ([1], [1]), ([1], [2])); +select minMap(val, cnt) from values ('val Array(DateTime(\'Asia/Istanbul\')), cnt Array(Int32)', ([1], [1]), ([1], [2])); select minMap(val, cnt) from values ('val Array(Decimal(10, 2)), cnt Array(Int16)', (['1.01'], [1]), (['1.01'], [2])); select minMap(val, cnt) from values ('val Array(Enum16(\'a\'=1)), cnt Array(Int16)', (['a'], [1]), (['a'], [2])); @@ -28,7 +28,7 @@ select maxMap(val, cnt) from values ('val Array(FixedString(1)), cnt Array(Fixed select maxMap(val, cnt) from values ('val Array(UInt64), cnt Array(UInt64)', ([1], [1]), ([1], [2])); select maxMap(val, cnt) from values ('val Array(Float64), cnt Array(Int8)', ([1], [1]), ([1], [2])); select maxMap(val, cnt) from values ('val Array(Date), cnt Array(Int16)', ([1], [1]), ([1], [2])); -select maxMap(val, cnt) from values ('val Array(DateTime(\'Europe/Moscow\')), cnt Array(Int32)', ([1], [1]), ([1], [2])); +select maxMap(val, cnt) from values ('val Array(DateTime(\'Asia/Istanbul\')), cnt Array(Int32)', ([1], [1]), ([1], [2])); select maxMap(val, cnt) from values ('val Array(Decimal(10, 2)), cnt Array(Int16)', (['1.01'], [1]), (['1.01'], [2])); select maxMap(val, cnt) from values ('val Array(Enum16(\'a\'=1)), cnt Array(Int16)', (['a'], [1]), (['a'], [2])); diff --git a/tests/queries/0_stateless/01307_orc_output_format.sh b/tests/queries/0_stateless/01307_orc_output_format.sh index 926398e55bd..b17792af051 100755 --- a/tests/queries/0_stateless/01307_orc_output_format.sh +++ b/tests/queries/0_stateless/01307_orc_output_format.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS orc"; -$CLICKHOUSE_CLIENT --query="CREATE TABLE orc (uint8 UInt8, uint16 UInt16, uint32 UInt32, uint64 UInt64, int8 Int8, int16 Int16, int32 Int32, int64 Int64, float Float32, double Float64, string String, fixed FixedString(4), date Date, datetime DateTime('Europe/Moscow'), decimal32 Decimal32(4), decimal64 Decimal64(10), decimal128 Decimal128(20), nullable Nullable(Int32)) ENGINE = Memory"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE orc (uint8 UInt8, uint16 UInt16, uint32 UInt32, uint64 UInt64, int8 Int8, int16 Int16, int32 Int32, int64 Int64, float Float32, double Float64, string String, fixed FixedString(4), date Date, datetime DateTime('Asia/Istanbul'), decimal32 Decimal32(4), decimal64 Decimal64(10), decimal128 Decimal128(20), nullable Nullable(Int32)) ENGINE = Memory"; $CLICKHOUSE_CLIENT --query="INSERT INTO orc VALUES (255, 65535, 4294967295, 100000000000, -128, -32768, -2147483648, -100000000000, 2.02, 10000.0000001, 'String', '2020', 18980, 1639872000, 1.0001, 1.00000001, 100000.00000000000001, 1), (4, 1234, 3244467295, 500000000000, -1, -256, -14741221, -7000000000, 100.1, 14321.032141201, 'Another string', '2000', 20000, 1839882000, 34.1234, 123123.123123123, 123123123.123123123123123, NULL), (42, 42, 42, 42, 42, 42, 42, 42, 42.42, 42.42, '42', '4242', 42, 42, 42.42, 42.42424242, 424242.42424242424242, 42)"; diff --git a/tests/queries/0_stateless/01379_with_fill_several_columns.sql b/tests/queries/0_stateless/01379_with_fill_several_columns.sql index 505b9e0f8e1..6bdf7d41b57 100644 --- a/tests/queries/0_stateless/01379_with_fill_several_columns.sql +++ b/tests/queries/0_stateless/01379_with_fill_several_columns.sql @@ -1,6 +1,6 @@ SELECT - toDate(toDateTime((number * 10) * 86400, 'Europe/Moscow')) AS d1, - toDate(toDateTime(number * 86400, 'Europe/Moscow')) AS d2, + toDate(toDateTime((number * 10) * 86400, 'Asia/Istanbul')) AS d1, + toDate(toDateTime(number * 86400, 'Asia/Istanbul')) AS d2, 'original' AS source FROM numbers(10) WHERE (number % 3) = 1 @@ -11,8 +11,8 @@ ORDER BY SELECT '==============='; SELECT - toDate(toDateTime((number * 10) * 86400, 'Europe/Moscow')) AS d1, - toDate(toDateTime(number * 86400, 'Europe/Moscow')) AS d2, + toDate(toDateTime((number * 10) * 86400, 'Asia/Istanbul')) AS d1, + toDate(toDateTime(number * 86400, 'Asia/Istanbul')) AS d2, 'original' AS source FROM numbers(10) WHERE (number % 3) = 1 diff --git a/tests/queries/0_stateless/01396_negative_datetime_saturate_to_zero.sql b/tests/queries/0_stateless/01396_negative_datetime_saturate_to_zero.sql index 2711ab3cceb..e52c2d3dd1b 100644 --- a/tests/queries/0_stateless/01396_negative_datetime_saturate_to_zero.sql +++ b/tests/queries/0_stateless/01396_negative_datetime_saturate_to_zero.sql @@ -1 +1 @@ -SELECT toTimeZone(now(), 'Europe/Moscow') > '1970-01-01 00:00:00'; +SELECT toTimeZone(now(), 'Asia/Istanbul') > '1970-01-01 00:00:00'; diff --git a/tests/queries/0_stateless/01414_low_cardinality_nullable.sql b/tests/queries/0_stateless/01414_low_cardinality_nullable.sql index c11e990cea8..871d74d7fb9 100644 --- a/tests/queries/0_stateless/01414_low_cardinality_nullable.sql +++ b/tests/queries/0_stateless/01414_low_cardinality_nullable.sql @@ -15,7 +15,7 @@ CREATE TABLE lc_nullable ( f64 Array(LowCardinality(Nullable(Float64))), date Array(LowCardinality(Nullable(Date))), - date_time Array(LowCardinality(Nullable(DateTime('Europe/Moscow')))), + date_time Array(LowCardinality(Nullable(DateTime('Asia/Istanbul')))), str Array(LowCardinality(Nullable(String))), fixed_string Array(LowCardinality(Nullable(FixedString(5)))) @@ -33,8 +33,8 @@ INSERT INTO lc_nullable SELECT groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, - groupArray(toDate(number, 'Europe/Moscow')) AS date, - groupArray(toDateTime(number, 'Europe/Moscow')) AS date_time, + groupArray(toDate(number, 'Asia/Istanbul')) AS date, + groupArray(toDateTime(number, 'Asia/Istanbul')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers LIMIT 15); @@ -51,8 +51,8 @@ INSERT INTO lc_nullable SELECT groupArray(toUInt64(num)) AS u64, groupArray(toFloat32(num)) AS f32, groupArray(toFloat64(num)) AS f64, - groupArray(toDate(num, 'Europe/Moscow')) AS date, - groupArray(toDateTime(num, 'Europe/Moscow')) AS date_time, + groupArray(toDate(num, 'Asia/Istanbul')) AS date, + groupArray(toDateTime(num, 'Asia/Istanbul')) AS date_time, groupArray(toString(num)) AS str, groupArray(toFixedString(toString(num), 5)) AS fixed_string FROM (SELECT negate(number) as num FROM system.numbers LIMIT 15); @@ -69,8 +69,8 @@ INSERT INTO lc_nullable SELECT groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, - groupArray(toDate(number, 'Europe/Moscow')) AS date, - groupArray(toDateTime(number, 'Europe/Moscow')) AS date_time, + groupArray(toDate(number, 'Asia/Istanbul')) AS date, + groupArray(toDateTime(number, 'Asia/Istanbul')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers WHERE number >= 5 LIMIT 15); @@ -87,8 +87,8 @@ INSERT INTO lc_nullable SELECT groupArray(toUInt64(number)) AS u64, groupArray(toFloat32(number)) AS f32, groupArray(toFloat64(number)) AS f64, - groupArray(toDate(number, 'Europe/Moscow')) AS date, - groupArray(toDateTime(number, 'Europe/Moscow')) AS date_time, + groupArray(toDate(number, 'Asia/Istanbul')) AS date, + groupArray(toDateTime(number, 'Asia/Istanbul')) AS date_time, groupArray(toString(number)) AS str, groupArray(toFixedString(toString(number), 5)) AS fixed_string FROM (SELECT number FROM system.numbers WHERE number >= 10 LIMIT 15); @@ -123,8 +123,8 @@ INSERT INTO lc_nullable SELECT [NULL, toUInt64(n)] AS u64, [NULL, toFloat32(n)] AS f32, [NULL, toFloat64(n)] AS f64, - [NULL, toDate(n, 'Europe/Moscow')] AS date, - [NULL, toDateTime(n, 'Europe/Moscow')] AS date_time, + [NULL, toDate(n, 'Asia/Istanbul')] AS date, + [NULL, toDateTime(n, 'Asia/Istanbul')] AS date_time, [NULL, toString(n)] AS str, [NULL, toFixedString(toString(n), 5)] AS fixed_string FROM (SELECT 100 as n); @@ -140,7 +140,7 @@ SELECT count() FROM lc_nullable WHERE has(u64, 1); SELECT count() FROM lc_nullable WHERE has(f32, 1); SELECT count() FROM lc_nullable WHERE has(f64, 1); SELECT count() FROM lc_nullable WHERE has(date, toDate('1970-01-02')); -SELECT count() FROM lc_nullable WHERE has(date_time, toDateTime('1970-01-01 03:00:01', 'Europe/Moscow')); +SELECT count() FROM lc_nullable WHERE has(date_time, toDateTime('1970-01-01 03:00:01', 'Asia/Istanbul')); SELECT count() FROM lc_nullable WHERE has(str, '1'); SELECT count() FROM lc_nullable WHERE has(fixed_string, toFixedString('1', 5)); @@ -168,7 +168,7 @@ SELECT count() FROM lc_nullable WHERE has(u64, 5); SELECT count() FROM lc_nullable WHERE has(f32, 5); SELECT count() FROM lc_nullable WHERE has(f64, 5); SELECT count() FROM lc_nullable WHERE has(date, toDate('1970-01-06')); -SELECT count() FROM lc_nullable WHERE has(date_time, toDateTime('1970-01-01 03:00:05', 'Europe/Moscow')); +SELECT count() FROM lc_nullable WHERE has(date_time, toDateTime('1970-01-01 03:00:05', 'Asia/Istanbul')); SELECT count() FROM lc_nullable WHERE has(str, '5'); SELECT count() FROM lc_nullable WHERE has(fixed_string, toFixedString('5', 5)); @@ -183,7 +183,7 @@ SELECT count() FROM lc_nullable WHERE has(u64, 10); SELECT count() FROM lc_nullable WHERE has(f32, 10); SELECT count() FROM lc_nullable WHERE has(f64, 10); SELECT count() FROM lc_nullable WHERE has(date, toDate('1970-01-11')); -SELECT count() FROM lc_nullable WHERE has(date_time, toDateTime('1970-01-01 03:00:10', 'Europe/Moscow')); +SELECT count() FROM lc_nullable WHERE has(date_time, toDateTime('1970-01-01 03:00:10', 'Asia/Istanbul')); SELECT count() FROM lc_nullable WHERE has(str, '10'); SELECT count() FROM lc_nullable WHERE has(fixed_string, toFixedString('10', 5)); @@ -213,7 +213,7 @@ SELECT count() FROM lc_nullable WHERE has(u64, 100); SELECT count() FROM lc_nullable WHERE has(f32, 100); SELECT count() FROM lc_nullable WHERE has(f64, 100); SELECT count() FROM lc_nullable WHERE has(date, toDate('1970-04-11')); -SELECT count() FROM lc_nullable WHERE has(date_time, toDateTime('1970-01-01 03:01:40', 'Europe/Moscow')); +SELECT count() FROM lc_nullable WHERE has(date_time, toDateTime('1970-01-01 03:01:40', 'Asia/Istanbul')); SELECT count() FROM lc_nullable WHERE has(str, '100'); SELECT count() FROM lc_nullable WHERE has(fixed_string, toFixedString('100', 5)); diff --git a/tests/queries/0_stateless/01432_parse_date_time_best_effort_timestamp.sql b/tests/queries/0_stateless/01432_parse_date_time_best_effort_timestamp.sql index 071fefe0403..58759c8585b 100644 --- a/tests/queries/0_stateless/01432_parse_date_time_best_effort_timestamp.sql +++ b/tests/queries/0_stateless/01432_parse_date_time_best_effort_timestamp.sql @@ -1,3 +1,3 @@ -SELECT parseDateTimeBestEffort('1596752940', 'Europe/Moscow'); -SELECT parseDateTimeBestEffort('100000000', 'Europe/Moscow'); -SELECT parseDateTimeBestEffort('20200807', 'Europe/Moscow'); +SELECT parseDateTimeBestEffort('1596752940', 'Asia/Istanbul'); +SELECT parseDateTimeBestEffort('100000000', 'Asia/Istanbul'); +SELECT parseDateTimeBestEffort('20200807', 'Asia/Istanbul'); diff --git a/tests/queries/0_stateless/01440_to_date_monotonicity.sql b/tests/queries/0_stateless/01440_to_date_monotonicity.sql index 8843d7ffca6..b4c4f98b223 100644 --- a/tests/queries/0_stateless/01440_to_date_monotonicity.sql +++ b/tests/queries/0_stateless/01440_to_date_monotonicity.sql @@ -1,11 +1,11 @@ DROP TABLE IF EXISTS tdm; DROP TABLE IF EXISTS tdm2; -CREATE TABLE tdm (x DateTime('Europe/Moscow')) ENGINE = MergeTree ORDER BY x SETTINGS write_final_mark = 0; +CREATE TABLE tdm (x DateTime('Asia/Istanbul')) ENGINE = MergeTree ORDER BY x SETTINGS write_final_mark = 0; INSERT INTO tdm VALUES (now()); -SELECT count(x) FROM tdm WHERE toDate(x) < toDate(now(), 'Europe/Moscow') SETTINGS max_rows_to_read = 1; +SELECT count(x) FROM tdm WHERE toDate(x) < toDate(now(), 'Asia/Istanbul') SETTINGS max_rows_to_read = 1; -SELECT toDate(-1), toDate(10000000000000, 'Europe/Moscow'), toDate(100), toDate(65536, 'UTC'), toDate(65535, 'Europe/Moscow'); -SELECT toDateTime(-1, 'Europe/Moscow'), toDateTime(10000000000000, 'Europe/Moscow'), toDateTime(1000, 'Europe/Moscow'); +SELECT toDate(-1), toDate(10000000000000, 'Asia/Istanbul'), toDate(100), toDate(65536, 'UTC'), toDate(65535, 'Asia/Istanbul'); +SELECT toDateTime(-1, 'Asia/Istanbul'), toDateTime(10000000000000, 'Asia/Istanbul'), toDateTime(1000, 'Asia/Istanbul'); CREATE TABLE tdm2 (timestamp UInt32) ENGINE = MergeTree ORDER BY timestamp SETTINGS index_granularity = 1; diff --git a/tests/queries/0_stateless/01442_date_time_with_params.reference b/tests/queries/0_stateless/01442_date_time_with_params.reference index 726e59d4d35..bc819d5f8fc 100644 --- a/tests/queries/0_stateless/01442_date_time_with_params.reference +++ b/tests/queries/0_stateless/01442_date_time_with_params.reference @@ -1,6 +1,6 @@ -2020-01-01 00:00:00 DateTime 2020-01-01 00:01:00 DateTime 2020-01-01 00:02:00.11 DateTime64(2) 2020-01-01 00:03:00 DateTime(\'Europe/Moscow\') 2020-01-01 00:04:00.220 DateTime64(3, \'Europe/Moscow\') 2020-01-01 00:05:00 DateTime 2020-01-01 00:06:00 DateTime(\'Europe/Moscow\') 2020-01-01 00:06:00 DateTime -2020-01-01 00:00:00 DateTime 2020-01-01 00:02:00.11 DateTime64(2) 2020-01-01 00:03:00 DateTime(\'Europe/Moscow\') 2020-01-01 00:04:00.220 DateTime64(3, \'Europe/Moscow\') 2020-01-01 00:05:00 DateTime -2020-01-01 00:00:00 DateTime 2020-01-01 00:02:00.11 DateTime64(2) 2020-01-01 00:03:00 DateTime(\'Europe/Moscow\') 2020-01-01 00:04:00.220 DateTime64(3, \'Europe/Moscow\') 2020-01-01 00:05:00 DateTime +2020-01-01 00:00:00 DateTime 2020-01-01 00:01:00 DateTime 2020-01-01 00:02:00.11 DateTime64(2) 2020-01-01 00:03:00 DateTime(\'Asia/Istanbul\') 2020-01-01 00:04:00.220 DateTime64(3, \'Asia/Istanbul\') 2020-01-01 00:05:00 DateTime 2020-01-01 00:06:00 DateTime(\'Asia/Istanbul\') 2020-01-01 00:06:00 DateTime +2020-01-01 00:00:00 DateTime 2020-01-01 00:02:00.11 DateTime64(2) 2020-01-01 00:03:00 DateTime(\'Asia/Istanbul\') 2020-01-01 00:04:00.220 DateTime64(3, \'Asia/Istanbul\') 2020-01-01 00:05:00 DateTime +2020-01-01 00:00:00 DateTime 2020-01-01 00:02:00.11 DateTime64(2) 2020-01-01 00:03:00 DateTime(\'Asia/Istanbul\') 2020-01-01 00:04:00.220 DateTime64(3, \'Asia/Istanbul\') 2020-01-01 00:05:00 DateTime 2020-01-01 00:00:00 DateTime parseDateTimeBestEffort 2020-05-14 03:37:03.000 DateTime64(3, \'UTC\') diff --git a/tests/queries/0_stateless/01442_date_time_with_params.sql b/tests/queries/0_stateless/01442_date_time_with_params.sql index 5a57aabdb0c..aeb9aa597ef 100644 --- a/tests/queries/0_stateless/01442_date_time_with_params.sql +++ b/tests/queries/0_stateless/01442_date_time_with_params.sql @@ -1,14 +1,14 @@ DROP TABLE IF EXISTS test; -CREATE TABLE test (a DateTime, b DateTime(), c DateTime(2), d DateTime('Europe/Moscow'), e DateTime(3, 'Europe/Moscow'), f DateTime32, g DateTime32('Europe/Moscow'), h DateTime(0)) ENGINE = MergeTree ORDER BY a; +CREATE TABLE test (a DateTime, b DateTime(), c DateTime(2), d DateTime('Asia/Istanbul'), e DateTime(3, 'Asia/Istanbul'), f DateTime32, g DateTime32('Asia/Istanbul'), h DateTime(0)) ENGINE = MergeTree ORDER BY a; INSERT INTO test VALUES('2020-01-01 00:00:00', '2020-01-01 00:01:00', '2020-01-01 00:02:00.11', '2020-01-01 00:03:00', '2020-01-01 00:04:00.22', '2020-01-01 00:05:00', '2020-01-01 00:06:00', '2020-01-01 00:06:00'); SELECT a, toTypeName(a), b, toTypeName(b), c, toTypeName(c), d, toTypeName(d), e, toTypeName(e), f, toTypeName(f), g, toTypeName(g), h, toTypeName(h) FROM test; -SELECT toDateTime('2020-01-01 00:00:00') AS a, toTypeName(a), toDateTime('2020-01-01 00:02:00.11', 2) AS b, toTypeName(b), toDateTime('2020-01-01 00:03:00', 'Europe/Moscow') AS c, toTypeName(c), toDateTime('2020-01-01 00:04:00.22', 3, 'Europe/Moscow') AS d, toTypeName(d), toDateTime('2020-01-01 00:05:00', 0) AS e, toTypeName(e); +SELECT toDateTime('2020-01-01 00:00:00') AS a, toTypeName(a), toDateTime('2020-01-01 00:02:00.11', 2) AS b, toTypeName(b), toDateTime('2020-01-01 00:03:00', 'Asia/Istanbul') AS c, toTypeName(c), toDateTime('2020-01-01 00:04:00.22', 3, 'Asia/Istanbul') AS d, toTypeName(d), toDateTime('2020-01-01 00:05:00', 0) AS e, toTypeName(e); -SELECT CAST('2020-01-01 00:00:00', 'DateTime') AS a, toTypeName(a), CAST('2020-01-01 00:02:00.11', 'DateTime(2)') AS b, toTypeName(b), CAST('2020-01-01 00:03:00', 'DateTime(\'Europe/Moscow\')') AS c, toTypeName(c), CAST('2020-01-01 00:04:00.22', 'DateTime(3, \'Europe/Moscow\')') AS d, toTypeName(d), CAST('2020-01-01 00:05:00', 'DateTime(0)') AS e, toTypeName(e); +SELECT CAST('2020-01-01 00:00:00', 'DateTime') AS a, toTypeName(a), CAST('2020-01-01 00:02:00.11', 'DateTime(2)') AS b, toTypeName(b), CAST('2020-01-01 00:03:00', 'DateTime(\'Asia/Istanbul\')') AS c, toTypeName(c), CAST('2020-01-01 00:04:00.22', 'DateTime(3, \'Asia/Istanbul\')') AS d, toTypeName(d), CAST('2020-01-01 00:05:00', 'DateTime(0)') AS e, toTypeName(e); SELECT toDateTime32('2020-01-01 00:00:00') AS a, toTypeName(a); diff --git a/tests/queries/0_stateless/01508_partition_pruning_long.queries b/tests/queries/0_stateless/01508_partition_pruning_long.queries index 786240145a9..0d64fc05f0f 100644 --- a/tests/queries/0_stateless/01508_partition_pruning_long.queries +++ b/tests/queries/0_stateless/01508_partition_pruning_long.queries @@ -2,20 +2,20 @@ DROP TABLE IF EXISTS tMM; DROP TABLE IF EXISTS tDD; DROP TABLE IF EXISTS sDD; DROP TABLE IF EXISTS xMM; -CREATE TABLE tMM(d DateTime('Europe/Moscow'), a Int64) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY tuple() SETTINGS index_granularity = 8192; +CREATE TABLE tMM(d DateTime('Asia/Istanbul'), a Int64) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY tuple() SETTINGS index_granularity = 8192; SYSTEM STOP MERGES tMM; -INSERT INTO tMM SELECT toDateTime('2020-08-16 00:00:00', 'Europe/Moscow') + number*60, number FROM numbers(5000); -INSERT INTO tMM SELECT toDateTime('2020-08-16 00:00:00', 'Europe/Moscow') + number*60, number FROM numbers(5000); -INSERT INTO tMM SELECT toDateTime('2020-09-01 00:00:00', 'Europe/Moscow') + number*60, number FROM numbers(5000); -INSERT INTO tMM SELECT toDateTime('2020-09-01 00:00:00', 'Europe/Moscow') + number*60, number FROM numbers(5000); -INSERT INTO tMM SELECT toDateTime('2020-10-01 00:00:00', 'Europe/Moscow') + number*60, number FROM numbers(5000); -INSERT INTO tMM SELECT toDateTime('2020-10-15 00:00:00', 'Europe/Moscow') + number*60, number FROM numbers(5000); +INSERT INTO tMM SELECT toDateTime('2020-08-16 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000); +INSERT INTO tMM SELECT toDateTime('2020-08-16 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000); +INSERT INTO tMM SELECT toDateTime('2020-09-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000); +INSERT INTO tMM SELECT toDateTime('2020-09-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000); +INSERT INTO tMM SELECT toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000); +INSERT INTO tMM SELECT toDateTime('2020-10-15 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000); -CREATE TABLE tDD(d DateTime('Europe/Moscow'),a Int) ENGINE = MergeTree PARTITION BY toYYYYMMDD(d) ORDER BY tuple() SETTINGS index_granularity = 8192; +CREATE TABLE tDD(d DateTime('Asia/Istanbul'),a Int) ENGINE = MergeTree PARTITION BY toYYYYMMDD(d) ORDER BY tuple() SETTINGS index_granularity = 8192; SYSTEM STOP MERGES tDD; -insert into tDD select toDateTime(toDate('2020-09-23'), 'Europe/Moscow'), number from numbers(10000) UNION ALL select toDateTime(toDateTime('2020-09-23 11:00:00', 'Europe/Moscow')), number from numbers(10000) UNION ALL select toDateTime(toDate('2020-09-24'), 'Europe/Moscow'), number from numbers(10000) UNION ALL select toDateTime(toDate('2020-09-25'), 'Europe/Moscow'), number from numbers(10000) UNION ALL select toDateTime(toDate('2020-08-15'), 'Europe/Moscow'), number from numbers(10000); +insert into tDD select toDateTime(toDate('2020-09-23'), 'Asia/Istanbul'), number from numbers(10000) UNION ALL select toDateTime(toDateTime('2020-09-23 11:00:00', 'Asia/Istanbul')), number from numbers(10000) UNION ALL select toDateTime(toDate('2020-09-24'), 'Asia/Istanbul'), number from numbers(10000) UNION ALL select toDateTime(toDate('2020-09-25'), 'Asia/Istanbul'), number from numbers(10000) UNION ALL select toDateTime(toDate('2020-08-15'), 'Asia/Istanbul'), number from numbers(10000); -CREATE TABLE sDD(d UInt64,a Int) ENGINE = MergeTree PARTITION BY toYYYYMM(toDate(intDiv(d,1000), 'Europe/Moscow')) ORDER BY tuple() SETTINGS index_granularity = 8192; +CREATE TABLE sDD(d UInt64,a Int) ENGINE = MergeTree PARTITION BY toYYYYMM(toDate(intDiv(d,1000), 'Asia/Istanbul')) ORDER BY tuple() SETTINGS index_granularity = 8192; SYSTEM STOP MERGES sDD; insert into sDD select (1597536000+number*60)*1000, number from numbers(5000); insert into sDD select (1597536000+number*60)*1000, number from numbers(5000); @@ -24,14 +24,14 @@ insert into sDD select (1598918400+number*60)*1000, number from numbers(5000); insert into sDD select (1601510400+number*60)*1000, number from numbers(5000); insert into sDD select (1602720000+number*60)*1000, number from numbers(5000); -CREATE TABLE xMM(d DateTime('Europe/Moscow'),a Int64, f Int64) ENGINE = MergeTree PARTITION BY (toYYYYMM(d), a) ORDER BY tuple() SETTINGS index_granularity = 8192; +CREATE TABLE xMM(d DateTime('Asia/Istanbul'),a Int64, f Int64) ENGINE = MergeTree PARTITION BY (toYYYYMM(d), a) ORDER BY tuple() SETTINGS index_granularity = 8192; SYSTEM STOP MERGES xMM; -INSERT INTO xMM SELECT toDateTime('2020-08-16 00:00:00', 'Europe/Moscow') + number*60, 1, number FROM numbers(5000); -INSERT INTO xMM SELECT toDateTime('2020-08-16 00:00:00', 'Europe/Moscow') + number*60, 2, number FROM numbers(5000); -INSERT INTO xMM SELECT toDateTime('2020-09-01 00:00:00', 'Europe/Moscow') + number*60, 3, number FROM numbers(5000); -INSERT INTO xMM SELECT toDateTime('2020-09-01 00:00:00', 'Europe/Moscow') + number*60, 2, number FROM numbers(5000); -INSERT INTO xMM SELECT toDateTime('2020-10-01 00:00:00', 'Europe/Moscow') + number*60, 1, number FROM numbers(5000); -INSERT INTO xMM SELECT toDateTime('2020-10-15 00:00:00', 'Europe/Moscow') + number*60, 1, number FROM numbers(5000); +INSERT INTO xMM SELECT toDateTime('2020-08-16 00:00:00', 'Asia/Istanbul') + number*60, 1, number FROM numbers(5000); +INSERT INTO xMM SELECT toDateTime('2020-08-16 00:00:00', 'Asia/Istanbul') + number*60, 2, number FROM numbers(5000); +INSERT INTO xMM SELECT toDateTime('2020-09-01 00:00:00', 'Asia/Istanbul') + number*60, 3, number FROM numbers(5000); +INSERT INTO xMM SELECT toDateTime('2020-09-01 00:00:00', 'Asia/Istanbul') + number*60, 2, number FROM numbers(5000); +INSERT INTO xMM SELECT toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul') + number*60, 1, number FROM numbers(5000); +INSERT INTO xMM SELECT toDateTime('2020-10-15 00:00:00', 'Asia/Istanbul') + number*60, 1, number FROM numbers(5000); SELECT '--------- tMM ----------------------------'; @@ -44,8 +44,8 @@ select uniqExact(_part), count() from tMM where toYYYYMMDD(d)=20200816; select uniqExact(_part), count() from tMM where toYYYYMMDD(d)=20201015; select uniqExact(_part), count() from tMM where toDate(d)='2020-10-15'; select uniqExact(_part), count() from tMM where d >= '2020-09-01 00:00:00' and d<'2020-10-15 00:00:00'; -select uniqExact(_part), count() from tMM where d >= '2020-01-16 00:00:00' and d < toDateTime('2021-08-17 00:00:00', 'Europe/Moscow'); -select uniqExact(_part), count() from tMM where d >= '2020-09-16 00:00:00' and d < toDateTime('2020-10-01 00:00:00', 'Europe/Moscow'); +select uniqExact(_part), count() from tMM where d >= '2020-01-16 00:00:00' and d < toDateTime('2021-08-17 00:00:00', 'Asia/Istanbul'); +select uniqExact(_part), count() from tMM where d >= '2020-09-16 00:00:00' and d < toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul'); select uniqExact(_part), count() from tMM where d >= '2020-09-12 00:00:00' and d < '2020-10-16 00:00:00'; select uniqExact(_part), count() from tMM where toStartOfDay(d) >= '2020-09-12 00:00:00'; select uniqExact(_part), count() from tMM where toStartOfDay(d) = '2020-09-01 00:00:00'; diff --git a/tests/queries/0_stateless/01508_partition_pruning_long.reference b/tests/queries/0_stateless/01508_partition_pruning_long.reference index 9cd208a336f..afdb4257505 100644 --- a/tests/queries/0_stateless/01508_partition_pruning_long.reference +++ b/tests/queries/0_stateless/01508_partition_pruning_long.reference @@ -35,11 +35,11 @@ select uniqExact(_part), count() from tMM where d >= '2020-09-01 00:00:00' and d 3 15000 Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges -select uniqExact(_part), count() from tMM where d >= '2020-01-16 00:00:00' and d < toDateTime('2021-08-17 00:00:00', 'Europe/Moscow'); +select uniqExact(_part), count() from tMM where d >= '2020-01-16 00:00:00' and d < toDateTime('2021-08-17 00:00:00', 'Asia/Istanbul'); 6 30000 Selected 6/6 parts by partition key, 6 parts by primary key, 6/6 marks by primary key, 6 marks to read from 6 ranges -select uniqExact(_part), count() from tMM where d >= '2020-09-16 00:00:00' and d < toDateTime('2020-10-01 00:00:00', 'Europe/Moscow'); +select uniqExact(_part), count() from tMM where d >= '2020-09-16 00:00:00' and d < toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul'); 0 0 Selected 0/6 parts by partition key, 0 parts by primary key, 0/0 marks by primary key, 0 marks to read from 0 ranges diff --git a/tests/queries/0_stateless/01516_date_time_output_format.sql b/tests/queries/0_stateless/01516_date_time_output_format.sql index 224d8ef1035..3c99d1bb81b 100644 --- a/tests/queries/0_stateless/01516_date_time_output_format.sql +++ b/tests/queries/0_stateless/01516_date_time_output_format.sql @@ -1,16 +1,16 @@ DROP TABLE IF EXISTS test_datetime; -CREATE TABLE test_datetime(timestamp DateTime('Europe/Moscow')) ENGINE=Log; +CREATE TABLE test_datetime(timestamp DateTime('Asia/Istanbul')) ENGINE=Log; INSERT INTO test_datetime VALUES ('2020-10-15 00:00:00'); SET date_time_output_format = 'simple'; SELECT timestamp FROM test_datetime; -SELECT formatDateTime(toDateTime('2020-10-15 00:00:00', 'Europe/Moscow'), '%Y-%m-%d %R:%S') as formatted_simple FROM test_datetime; +SELECT formatDateTime(toDateTime('2020-10-15 00:00:00', 'Asia/Istanbul'), '%Y-%m-%d %R:%S') as formatted_simple FROM test_datetime; SET date_time_output_format = 'iso'; SELECT timestamp FROM test_datetime; -SELECT formatDateTime(toDateTime('2020-10-15 00:00:00', 'Europe/Moscow'), '%Y-%m-%dT%R:%SZ', 'UTC') as formatted_iso FROM test_datetime;; +SELECT formatDateTime(toDateTime('2020-10-15 00:00:00', 'Asia/Istanbul'), '%Y-%m-%dT%R:%SZ', 'UTC') as formatted_iso FROM test_datetime;; SET date_time_output_format = 'unix_timestamp'; SELECT timestamp FROM test_datetime; @@ -19,7 +19,7 @@ SELECT toUnixTimestamp(timestamp) FROM test_datetime; SET date_time_output_format = 'simple'; DROP TABLE test_datetime; -CREATE TABLE test_datetime(timestamp DateTime64(3, 'Europe/Moscow')) Engine=Log; +CREATE TABLE test_datetime(timestamp DateTime64(3, 'Asia/Istanbul')) Engine=Log; INSERT INTO test_datetime VALUES ('2020-10-15 00:00:00'), (1602709200123); diff --git a/tests/queries/0_stateless/01582_any_join_supertype.sql b/tests/queries/0_stateless/01582_any_join_supertype.sql index 6b06d78c83c..9cd7b4397ab 100644 --- a/tests/queries/0_stateless/01582_any_join_supertype.sql +++ b/tests/queries/0_stateless/01582_any_join_supertype.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS foo; DROP TABLE IF EXISTS bar; -CREATE TABLE foo (server_date Date, server_time Datetime('Europe/Moscow'), dimension_1 String) ENGINE = MergeTree() PARTITION BY toYYYYMM(server_date) ORDER BY (server_date); +CREATE TABLE foo (server_date Date, server_time Datetime('Asia/Istanbul'), dimension_1 String) ENGINE = MergeTree() PARTITION BY toYYYYMM(server_date) ORDER BY (server_date); CREATE TABLE bar (server_date Date, dimension_1 String) ENGINE = MergeTree() PARTITION BY toYYYYMM(server_date) ORDER BY (server_date); INSERT INTO foo VALUES ('2020-01-01', '2020-01-01 12:00:00', 'test1'), ('2020-01-01', '2020-01-01 13:00:00', 'test2'); diff --git a/tests/queries/0_stateless/01615_two_args_function_index_fix.sql b/tests/queries/0_stateless/01615_two_args_function_index_fix.sql index dd2bde2eafc..6128bdfcdfb 100644 --- a/tests/queries/0_stateless/01615_two_args_function_index_fix.sql +++ b/tests/queries/0_stateless/01615_two_args_function_index_fix.sql @@ -1,6 +1,6 @@ drop table if exists bad_date_time; -create table bad_date_time (time Datetime('Europe/Moscow'), count UInt16) Engine = MergeTree() ORDER BY (time); +create table bad_date_time (time Datetime('Asia/Istanbul'), count UInt16) Engine = MergeTree() ORDER BY (time); insert into bad_date_time values('2020-12-20 20:59:52', 1), ('2020-12-20 21:59:52', 1), ('2020-12-20 01:59:52', 1); diff --git a/tests/queries/0_stateless/01676_reinterpret_as.sql b/tests/queries/0_stateless/01676_reinterpret_as.sql index e8c2a0b1373..cc52859724d 100644 --- a/tests/queries/0_stateless/01676_reinterpret_as.sql +++ b/tests/queries/0_stateless/01676_reinterpret_as.sql @@ -30,8 +30,8 @@ SELECT reinterpret(a, 'String'), reinterpretAsString(a), reinterpretAsUInt8('11' SELECT reinterpret(a, 'String'), reinterpretAsString(a), reinterpretAsUInt16('11') as a; SELECT 'Dates'; SELECT reinterpret(0, 'Date'), reinterpret('', 'Date'); -SELECT reinterpret(0, 'DateTime(''Europe/Moscow'')'), reinterpret('', 'DateTime(''Europe/Moscow'')'); -SELECT reinterpret(0, 'DateTime64(3, ''Europe/Moscow'')'), reinterpret('', 'DateTime64(3, ''Europe/Moscow'')'); +SELECT reinterpret(0, 'DateTime(''Asia/Istanbul'')'), reinterpret('', 'DateTime(''Asia/Istanbul'')'); +SELECT reinterpret(0, 'DateTime64(3, ''Asia/Istanbul'')'), reinterpret('', 'DateTime64(3, ''Asia/Istanbul'')'); SELECT 'Decimals'; SELECT reinterpret(toDecimal32(5, 2), 'Decimal32(2)'), reinterpret('1', 'Decimal32(2)'); SELECT reinterpret(toDecimal64(5, 2), 'Decimal64(2)'), reinterpret('1', 'Decimal64(2)');; diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.reference b/tests/queries/0_stateless/01691_DateTime64_clamp.reference index 41a8d653a3f..7b3b9ae04d6 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.reference +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.reference @@ -1,27 +1,27 @@ -- { echo } -- These values are within the extended range of DateTime64 [1925-01-01, 2284-01-01) -SELECT toTimeZone(toDateTime(-2, 2), 'Europe/Moscow'); +SELECT toTimeZone(toDateTime(-2, 2), 'Asia/Istanbul'); 1970-01-01 02:59:58.00 -SELECT toDateTime64(-2, 2, 'Europe/Moscow'); +SELECT toDateTime64(-2, 2, 'Asia/Istanbul'); 1970-01-01 02:59:58.00 -SELECT CAST(-1 AS DateTime64(0, 'Europe/Moscow')); +SELECT CAST(-1 AS DateTime64(0, 'Asia/Istanbul')); 1970-01-01 02:59:59 -SELECT CAST('2020-01-01 00:00:00.3' AS DateTime64(0, 'Europe/Moscow')); +SELECT CAST('2020-01-01 00:00:00.3' AS DateTime64(0, 'Asia/Istanbul')); 2020-01-01 00:00:00 -SELECT toDateTime64(bitShiftLeft(toUInt64(1), 33), 2, 'Europe/Moscow') FORMAT Null; -SELECT toTimeZone(toDateTime(-2., 2), 'Europe/Moscow'); +SELECT toDateTime64(bitShiftLeft(toUInt64(1), 33), 2, 'Asia/Istanbul') FORMAT Null; +SELECT toTimeZone(toDateTime(-2., 2), 'Asia/Istanbul'); 1970-01-01 03:00:00.00 -SELECT toDateTime64(-2., 2, 'Europe/Moscow'); +SELECT toDateTime64(-2., 2, 'Asia/Istanbul'); 1970-01-01 03:00:00.00 -SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow'); +SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Asia/Istanbul'); 2106-02-07 09:28:16.00 -SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; +SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Asia/Istanbul') FORMAT Null; -- These are outsize of extended range and hence clamped -SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1), 35), 2, 'Europe/Moscow'); +SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1), 35), 2, 'Asia/Istanbul'); 1925-01-01 02:00:00.00 -SELECT CAST(-1 * bitShiftLeft(toUInt64(1), 35) AS DateTime64(3, 'Europe/Moscow')); +SELECT CAST(-1 * bitShiftLeft(toUInt64(1), 35) AS DateTime64(3, 'Asia/Istanbul')); 1925-01-01 02:00:00.000 -SELECT CAST(bitShiftLeft(toUInt64(1), 35) AS DateTime64(3, 'Europe/Moscow')); +SELECT CAST(bitShiftLeft(toUInt64(1), 35) AS DateTime64(3, 'Asia/Istanbul')); 2282-12-31 03:00:00.000 -SELECT toDateTime64(bitShiftLeft(toUInt64(1), 35), 2, 'Europe/Moscow'); +SELECT toDateTime64(bitShiftLeft(toUInt64(1), 35), 2, 'Asia/Istanbul'); 2282-12-31 03:00:00.00 diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.sql b/tests/queries/0_stateless/01691_DateTime64_clamp.sql index 2786d9c1c09..b7077aff1f7 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.sql +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.sql @@ -1,17 +1,17 @@ -- { echo } -- These values are within the extended range of DateTime64 [1925-01-01, 2284-01-01) -SELECT toTimeZone(toDateTime(-2, 2), 'Europe/Moscow'); -SELECT toDateTime64(-2, 2, 'Europe/Moscow'); -SELECT CAST(-1 AS DateTime64(0, 'Europe/Moscow')); -SELECT CAST('2020-01-01 00:00:00.3' AS DateTime64(0, 'Europe/Moscow')); -SELECT toDateTime64(bitShiftLeft(toUInt64(1), 33), 2, 'Europe/Moscow') FORMAT Null; -SELECT toTimeZone(toDateTime(-2., 2), 'Europe/Moscow'); -SELECT toDateTime64(-2., 2, 'Europe/Moscow'); -SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow'); -SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; +SELECT toTimeZone(toDateTime(-2, 2), 'Asia/Istanbul'); +SELECT toDateTime64(-2, 2, 'Asia/Istanbul'); +SELECT CAST(-1 AS DateTime64(0, 'Asia/Istanbul')); +SELECT CAST('2020-01-01 00:00:00.3' AS DateTime64(0, 'Asia/Istanbul')); +SELECT toDateTime64(bitShiftLeft(toUInt64(1), 33), 2, 'Asia/Istanbul') FORMAT Null; +SELECT toTimeZone(toDateTime(-2., 2), 'Asia/Istanbul'); +SELECT toDateTime64(-2., 2, 'Asia/Istanbul'); +SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Asia/Istanbul'); +SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Asia/Istanbul') FORMAT Null; -- These are outsize of extended range and hence clamped -SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1), 35), 2, 'Europe/Moscow'); -SELECT CAST(-1 * bitShiftLeft(toUInt64(1), 35) AS DateTime64(3, 'Europe/Moscow')); -SELECT CAST(bitShiftLeft(toUInt64(1), 35) AS DateTime64(3, 'Europe/Moscow')); -SELECT toDateTime64(bitShiftLeft(toUInt64(1), 35), 2, 'Europe/Moscow'); +SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1), 35), 2, 'Asia/Istanbul'); +SELECT CAST(-1 * bitShiftLeft(toUInt64(1), 35) AS DateTime64(3, 'Asia/Istanbul')); +SELECT CAST(bitShiftLeft(toUInt64(1), 35) AS DateTime64(3, 'Asia/Istanbul')); +SELECT toDateTime64(bitShiftLeft(toUInt64(1), 35), 2, 'Asia/Istanbul'); diff --git a/tests/queries/0_stateless/01692_DateTime64_from_DateTime.sql b/tests/queries/0_stateless/01692_DateTime64_from_DateTime.sql index fac0c341007..c08062a456c 100644 --- a/tests/queries/0_stateless/01692_DateTime64_from_DateTime.sql +++ b/tests/queries/0_stateless/01692_DateTime64_from_DateTime.sql @@ -1,7 +1,7 @@ -select toDateTime64(toDateTime(1, 'Europe/Moscow'), 2); +select toDateTime64(toDateTime(1, 'Asia/Istanbul'), 2); select toDateTime64(toDate(1), 2) FORMAT Null; -- Unknown timezone select toDateTime64(toDateTime(1), 2) FORMAT Null; -- Unknown timezone -select toDateTime64(toDateTime(1), 2, 'Europe/Moscow'); -select toDateTime64(toDate(1), 2, 'Europe/Moscow'); +select toDateTime64(toDateTime(1), 2, 'Asia/Istanbul'); +select toDateTime64(toDate(1), 2, 'Asia/Istanbul'); select toDateTime64(toDateTime(1), 2, 'GMT'); select toDateTime64(toDate(1), 2, 'GMT'); diff --git a/tests/queries/0_stateless/01698_fix_toMinute.reference b/tests/queries/0_stateless/01698_fix_toMinute.reference index 7675aad3a57..eb1f7eb9ca1 100644 --- a/tests/queries/0_stateless/01698_fix_toMinute.reference +++ b/tests/queries/0_stateless/01698_fix_toMinute.reference @@ -19,6 +19,6 @@ Check the bug causing situation: the special Australia/Lord_Howe time zone. toDa 1554569400 2019-04-07 03:20:00 2019-04-07 03:20:00 1554570000 2019-04-07 03:30:00 2019-04-07 03:30:00 1554570600 2019-04-07 03:40:00 2019-04-07 03:40:00 -4 days test in batch comparing with manually computation result for Europe/Moscow whose timezone epoc is of whole hour: +4 days test in batch comparing with manually computation result for Asia/Istanbul whose timezone epoc is of whole hour: 4 days test in batch comparing with manually computation result for Asia/Tehran whose timezone epoc is of half hour: 4 days test in batch comparing with manually computation result for Australia/Lord_Howe whose timezone epoc is of half hour and also its DST offset is half hour: diff --git a/tests/queries/0_stateless/01698_fix_toMinute.sql b/tests/queries/0_stateless/01698_fix_toMinute.sql index f582806719d..4d11efa901d 100644 --- a/tests/queries/0_stateless/01698_fix_toMinute.sql +++ b/tests/queries/0_stateless/01698_fix_toMinute.sql @@ -3,9 +3,9 @@ SELECT 'Check the bug causing situation: the special Australia/Lord_Howe time zo SELECT toUnixTimestamp(x) as tt, (toDateTime('2019-04-07 01:00:00', 'Australia/Lord_Howe') + INTERVAL number * 600 SECOND) AS x, toString(x) as xx FROM numbers(20); /* The Batch Part. Test period is whole 4 days*/ -SELECT '4 days test in batch comparing with manually computation result for Europe/Moscow whose timezone epoc is of whole hour:'; -SELECT toUnixTimestamp(x) as tt, (toDateTime('1981-04-01 00:00:00', 'Europe/Moscow') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; -SELECT toUnixTimestamp(x) as tt, (toDateTime('1981-09-30 00:00:00', 'Europe/Moscow') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; +SELECT '4 days test in batch comparing with manually computation result for Asia/Istanbul whose timezone epoc is of whole hour:'; +SELECT toUnixTimestamp(x) as tt, (toDateTime('1981-04-01 00:00:00', 'Asia/Istanbul') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; +SELECT toUnixTimestamp(x) as tt, (toDateTime('1981-09-30 00:00:00', 'Asia/Istanbul') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; SELECT '4 days test in batch comparing with manually computation result for Asia/Tehran whose timezone epoc is of half hour:'; SELECT toUnixTimestamp(x) as tt, (toDateTime('2020-03-21 00:00:00', 'Asia/Tehran') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; diff --git a/tests/queries/0_stateless/01699_timezoneOffset.reference b/tests/queries/0_stateless/01699_timezoneOffset.reference index a1cc6391e6f..860829f0ce6 100644 --- a/tests/queries/0_stateless/01699_timezoneOffset.reference +++ b/tests/queries/0_stateless/01699_timezoneOffset.reference @@ -1,4 +1,4 @@ -DST boundary test for Europe/Moscow: +DST boundary test for Asia/Istanbul: 0 1981-04-01 22:40:00 14400 354998400 1 1981-04-01 22:50:00 14400 354999000 2 1981-04-01 23:00:00 14400 354999600 @@ -70,7 +70,7 @@ DST boundary test for Australia/Lord_Howe: 15 2019-04-07 03:00:00 37800 1554568200 16 2019-04-07 03:10:00 37800 1554568800 17 2019-04-07 03:20:00 37800 1554569400 -4 days test in batch comparing with manually computation result for Europe/Moscow: +4 days test in batch comparing with manually computation result for Asia/Istanbul: 4 days test in batch comparing with manually computation result for Asia/Tehran: 4 days test in batch comparing with manually computation result for Australia/Lord_Howe Moscow DST Years: diff --git a/tests/queries/0_stateless/01699_timezoneOffset.sql b/tests/queries/0_stateless/01699_timezoneOffset.sql index 8cabb23c4de..f9e6c2db970 100644 --- a/tests/queries/0_stateless/01699_timezoneOffset.sql +++ b/tests/queries/0_stateless/01699_timezoneOffset.sql @@ -1,8 +1,8 @@ /* Test the DST(daylight saving time) offset changing boundary*/ -SELECT 'DST boundary test for Europe/Moscow:'; -SELECT number,(toDateTime('1981-04-01 22:40:00', 'Europe/Moscow') + INTERVAL number * 600 SECOND) AS k, timezoneOffset(k) AS t, toUnixTimestamp(k) as s FROM numbers(4); -SELECT number,(toDateTime('1981-09-30 23:00:00', 'Europe/Moscow') + INTERVAL number * 600 SECOND) AS k, timezoneOffset(k) AS t, toUnixTimestamp(k) as s FROM numbers(18); +SELECT 'DST boundary test for Asia/Istanbul:'; +SELECT number,(toDateTime('1981-04-01 22:40:00', 'Asia/Istanbul') + INTERVAL number * 600 SECOND) AS k, timezoneOffset(k) AS t, toUnixTimestamp(k) as s FROM numbers(4); +SELECT number,(toDateTime('1981-09-30 23:00:00', 'Asia/Istanbul') + INTERVAL number * 600 SECOND) AS k, timezoneOffset(k) AS t, toUnixTimestamp(k) as s FROM numbers(18); SELECT 'DST boundary test for Asia/Tehran:'; SELECT number,(toDateTime('2020-03-21 22:40:00', 'Asia/Tehran') + INTERVAL number * 600 SECOND) AS k, timezoneOffset(k) AS t, toUnixTimestamp(k) as s FROM numbers(4); @@ -18,9 +18,9 @@ SELECT number,(toDateTime('2019-04-07 01:00:00', 'Australia/Lord_Howe') + INTERV /* The Batch Part. Test period is whole 4 days*/ -SELECT '4 days test in batch comparing with manually computation result for Europe/Moscow:'; -SELECT toUnixTimestamp(x) as tt, (toDateTime('1981-04-01 00:00:00', 'Europe/Moscow') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; -SELECT toUnixTimestamp(x) as tt, (toDateTime('1981-09-30 00:00:00', 'Europe/Moscow') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; +SELECT '4 days test in batch comparing with manually computation result for Asia/Istanbul:'; +SELECT toUnixTimestamp(x) as tt, (toDateTime('1981-04-01 00:00:00', 'Asia/Istanbul') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; +SELECT toUnixTimestamp(x) as tt, (toDateTime('1981-09-30 00:00:00', 'Asia/Istanbul') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; SELECT '4 days test in batch comparing with manually computation result for Asia/Tehran:'; SELECT toUnixTimestamp(x) as tt, (toDateTime('2020-03-21 00:00:00', 'Asia/Tehran') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; @@ -34,9 +34,9 @@ SELECT toUnixTimestamp(x) as tt, (toDateTime('2019-04-07 01:00:00', 'Australia/L /* Find all the years had followed DST during given period*/ SELECT 'Moscow DST Years:'; -SELECT number, (toDateTime('1970-06-01 00:00:00', 'Europe/Moscow') + INTERVAL number YEAR) AS DST_Y, timezoneOffset(DST_Y) AS t FROM numbers(51) where t != 10800; +SELECT number, (toDateTime('1970-06-01 00:00:00', 'Asia/Istanbul') + INTERVAL number YEAR) AS DST_Y, timezoneOffset(DST_Y) AS t FROM numbers(51) where t != 10800; SELECT 'Moscow DST Years with perment DST from 2011-2014:'; -SELECT min((toDateTime('2011-01-01 00:00:00', 'Europe/Moscow') + INTERVAL number DAY) as day) as start, max(day) as end, count(1), concat(toString(toYear(day)),'_',toString(timezoneOffset(day)))as DST from numbers(365*4+1) group by DST order by start; +SELECT min((toDateTime('2011-01-01 00:00:00', 'Asia/Istanbul') + INTERVAL number DAY) as day) as start, max(day) as end, count(1), concat(toString(toYear(day)),'_',toString(timezoneOffset(day)))as DST from numbers(365*4+1) group by DST order by start; SELECT 'Tehran DST Years:'; SELECT number, (toDateTime('1970-06-01 00:00:00', 'Asia/Tehran') + INTERVAL number YEAR) AS DST_Y, timezoneOffset(DST_Y) AS t FROM numbers(51) where t != 12600; diff --git a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.sql b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.sql index f51a1bb2280..b0dbd1dfc84 100644 --- a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.sql +++ b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.sql @@ -1,4 +1,4 @@ -SELECT toString(toDateTime('-922337203.6854775808', 1, 'Europe/Moscow')); -SELECT toString(toDateTime('9922337203.6854775808', 1, 'Europe/Moscow')); -SELECT toDateTime64(CAST('10000000000.1' AS Decimal64(1)), 1, 'Europe/Moscow'); -SELECT toDateTime64(CAST('-10000000000.1' AS Decimal64(1)), 1, 'Europe/Moscow'); +SELECT toString(toDateTime('-922337203.6854775808', 1, 'Asia/Istanbul')); +SELECT toString(toDateTime('9922337203.6854775808', 1, 'Asia/Istanbul')); +SELECT toDateTime64(CAST('10000000000.1' AS Decimal64(1)), 1, 'Asia/Istanbul'); +SELECT toDateTime64(CAST('-10000000000.1' AS Decimal64(1)), 1, 'Asia/Istanbul'); diff --git a/tests/queries/0_stateless/01732_more_consistent_datetime64_parsing.sql b/tests/queries/0_stateless/01732_more_consistent_datetime64_parsing.sql index 88859177a92..7e7fe3f2e16 100644 --- a/tests/queries/0_stateless/01732_more_consistent_datetime64_parsing.sql +++ b/tests/queries/0_stateless/01732_more_consistent_datetime64_parsing.sql @@ -5,7 +5,7 @@ INSERT INTO t VALUES (3, '1111111111222'); INSERT INTO t VALUES (4, '1111111111.222'); SELECT * FROM t ORDER BY i; -SELECT toDateTime64(1111111111.222, 3, 'Europe/Moscow'); -SELECT toDateTime64('1111111111.222', 3, 'Europe/Moscow'); -SELECT toDateTime64('1111111111222', 3, 'Europe/Moscow'); -SELECT ignore(toDateTime64(1111111111222, 3, 'Europe/Moscow')); -- This gives somewhat correct but unexpected result +SELECT toDateTime64(1111111111.222, 3, 'Asia/Istanbul'); +SELECT toDateTime64('1111111111.222', 3, 'Asia/Istanbul'); +SELECT toDateTime64('1111111111222', 3, 'Asia/Istanbul'); +SELECT ignore(toDateTime64(1111111111222, 3, 'Asia/Istanbul')); -- This gives somewhat correct but unexpected result diff --git a/tests/queries/0_stateless/01734_datetime64_from_float.sql b/tests/queries/0_stateless/01734_datetime64_from_float.sql index 416638a4a73..bb837c681e3 100644 --- a/tests/queries/0_stateless/01734_datetime64_from_float.sql +++ b/tests/queries/0_stateless/01734_datetime64_from_float.sql @@ -1,3 +1,3 @@ -SELECT CAST(1111111111.222 AS DateTime64(3, 'Europe/Moscow')); -SELECT toDateTime(1111111111.222, 3, 'Europe/Moscow'); -SELECT toDateTime64(1111111111.222, 3, 'Europe/Moscow'); +SELECT CAST(1111111111.222 AS DateTime64(3, 'Asia/Istanbul')); +SELECT toDateTime(1111111111.222, 3, 'Asia/Istanbul'); +SELECT toDateTime64(1111111111.222, 3, 'Asia/Istanbul'); diff --git a/tests/queries/0_stateless/01761_round_year_bounds.sql b/tests/queries/0_stateless/01761_round_year_bounds.sql index fed12c55568..57f421d155e 100644 --- a/tests/queries/0_stateless/01761_round_year_bounds.sql +++ b/tests/queries/0_stateless/01761_round_year_bounds.sql @@ -1 +1 @@ -SELECT toStartOfInterval(toDateTime(-9223372036854775808), toIntervalYear(100), 'Europe/Moscow') FORMAT Null; +SELECT toStartOfInterval(toDateTime(-9223372036854775808), toIntervalYear(100), 'Asia/Istanbul') FORMAT Null; diff --git a/tests/queries/0_stateless/01769_extended_range_2.sql b/tests/queries/0_stateless/01769_extended_range_2.sql index a2570c9397b..0b1319ddaea 100644 --- a/tests/queries/0_stateless/01769_extended_range_2.sql +++ b/tests/queries/0_stateless/01769_extended_range_2.sql @@ -1,3 +1,3 @@ SELECT toDateTime64('1969-12-31 18:00:12', 0, 'America/Phoenix'); SELECT toDateTime64('1969-12-30 18:00:12', 0, 'America/Phoenix'); -SELECT toDateTime64('1969-12-31 18:00:12', 0, 'Europe/Moscow'); +SELECT toDateTime64('1969-12-31 18:00:12', 0, 'Asia/Istanbul'); diff --git a/tests/queries/0_stateless/01772_to_start_of_hour_align.sql b/tests/queries/0_stateless/01772_to_start_of_hour_align.sql index 6d1bb460f90..5dbf6a97e57 100644 --- a/tests/queries/0_stateless/01772_to_start_of_hour_align.sql +++ b/tests/queries/0_stateless/01772_to_start_of_hour_align.sql @@ -13,8 +13,8 @@ SELECT toStartOfInterval(toDateTime('2021-03-23 13:58:00', 'Asia/Kolkata'), INTE -- In case of timezone shifts, rounding is performed to the hour number on "wall clock" time. -- The intervals may become shorter or longer due to time shifts. For example, the three hour interval may actually last two hours. -- If the same hour number on "wall clock" time correspond to multiple time points due to shifting backwards, the unspecified time point is selected among the candidates. -SELECT toDateTime('2010-03-28 00:00:00', 'Europe/Moscow') + INTERVAL 15 * number MINUTE AS src, toStartOfInterval(src, INTERVAL 2 HOUR) AS rounded, toUnixTimestamp(src) AS t FROM numbers(20); -SELECT toDateTime('2010-10-31 00:00:00', 'Europe/Moscow') + INTERVAL 15 * number MINUTE AS src, toStartOfInterval(src, INTERVAL 2 HOUR) AS rounded, toUnixTimestamp(src) AS t FROM numbers(20); +SELECT toDateTime('2010-03-28 00:00:00', 'Asia/Istanbul') + INTERVAL 15 * number MINUTE AS src, toStartOfInterval(src, INTERVAL 2 HOUR) AS rounded, toUnixTimestamp(src) AS t FROM numbers(20); +SELECT toDateTime('2010-10-31 00:00:00', 'Asia/Istanbul') + INTERVAL 15 * number MINUTE AS src, toStartOfInterval(src, INTERVAL 2 HOUR) AS rounded, toUnixTimestamp(src) AS t FROM numbers(20); -- And this should work even for non whole number of hours shifts. SELECT toDateTime('2020-04-05 00:00:00', 'Australia/Lord_Howe') + INTERVAL 15 * number MINUTE AS src, toStartOfInterval(src, INTERVAL 2 HOUR) AS rounded, toUnixTimestamp(src) AS t FROM numbers(20); diff --git a/tests/queries/0_stateless/01802_formatDateTime_DateTime64_century.reference b/tests/queries/0_stateless/01802_formatDateTime_DateTime64_century.reference index 75c114cdd74..bf717d7da0b 100644 --- a/tests/queries/0_stateless/01802_formatDateTime_DateTime64_century.reference +++ b/tests/queries/0_stateless/01802_formatDateTime_DateTime64_century.reference @@ -1,27 +1,27 @@ -- { echo } -SELECT formatDateTime(toDateTime64('1935-12-12 12:12:12', 0, 'Europe/Moscow'), '%C'); +SELECT formatDateTime(toDateTime64('1935-12-12 12:12:12', 0, 'Asia/Istanbul'), '%C'); 19 -SELECT formatDateTime(toDateTime64('1969-12-12 12:12:12', 0, 'Europe/Moscow'), '%C'); +SELECT formatDateTime(toDateTime64('1969-12-12 12:12:12', 0, 'Asia/Istanbul'), '%C'); 19 -SELECT formatDateTime(toDateTime64('1989-12-12 12:12:12', 0, 'Europe/Moscow'), '%C'); +SELECT formatDateTime(toDateTime64('1989-12-12 12:12:12', 0, 'Asia/Istanbul'), '%C'); 19 -SELECT formatDateTime(toDateTime64('2019-09-16 19:20:12', 0, 'Europe/Moscow'), '%C'); +SELECT formatDateTime(toDateTime64('2019-09-16 19:20:12', 0, 'Asia/Istanbul'), '%C'); 20 -SELECT formatDateTime(toDateTime64('2105-12-12 12:12:12', 0, 'Europe/Moscow'), '%C'); +SELECT formatDateTime(toDateTime64('2105-12-12 12:12:12', 0, 'Asia/Istanbul'), '%C'); 21 -SELECT formatDateTime(toDateTime64('2205-12-12 12:12:12', 0, 'Europe/Moscow'), '%C'); +SELECT formatDateTime(toDateTime64('2205-12-12 12:12:12', 0, 'Asia/Istanbul'), '%C'); 22 -- non-zero scale -SELECT formatDateTime(toDateTime64('1935-12-12 12:12:12', 6, 'Europe/Moscow'), '%C'); +SELECT formatDateTime(toDateTime64('1935-12-12 12:12:12', 6, 'Asia/Istanbul'), '%C'); 19 -SELECT formatDateTime(toDateTime64('1969-12-12 12:12:12', 6, 'Europe/Moscow'), '%C'); +SELECT formatDateTime(toDateTime64('1969-12-12 12:12:12', 6, 'Asia/Istanbul'), '%C'); 19 -SELECT formatDateTime(toDateTime64('1989-12-12 12:12:12', 6, 'Europe/Moscow'), '%C'); +SELECT formatDateTime(toDateTime64('1989-12-12 12:12:12', 6, 'Asia/Istanbul'), '%C'); 19 -SELECT formatDateTime(toDateTime64('2019-09-16 19:20:12', 0, 'Europe/Moscow'), '%C'); +SELECT formatDateTime(toDateTime64('2019-09-16 19:20:12', 0, 'Asia/Istanbul'), '%C'); 20 -SELECT formatDateTime(toDateTime64('2105-12-12 12:12:12', 6, 'Europe/Moscow'), '%C'); +SELECT formatDateTime(toDateTime64('2105-12-12 12:12:12', 6, 'Asia/Istanbul'), '%C'); 21 -SELECT formatDateTime(toDateTime64('2205-01-12 12:12:12', 6, 'Europe/Moscow'), '%C'); +SELECT formatDateTime(toDateTime64('2205-01-12 12:12:12', 6, 'Asia/Istanbul'), '%C'); 22 diff --git a/tests/queries/0_stateless/01802_formatDateTime_DateTime64_century.sql b/tests/queries/0_stateless/01802_formatDateTime_DateTime64_century.sql index e368f45cbda..712afd28cd6 100644 --- a/tests/queries/0_stateless/01802_formatDateTime_DateTime64_century.sql +++ b/tests/queries/0_stateless/01802_formatDateTime_DateTime64_century.sql @@ -1,16 +1,16 @@ -- { echo } -SELECT formatDateTime(toDateTime64('1935-12-12 12:12:12', 0, 'Europe/Moscow'), '%C'); -SELECT formatDateTime(toDateTime64('1969-12-12 12:12:12', 0, 'Europe/Moscow'), '%C'); -SELECT formatDateTime(toDateTime64('1989-12-12 12:12:12', 0, 'Europe/Moscow'), '%C'); -SELECT formatDateTime(toDateTime64('2019-09-16 19:20:12', 0, 'Europe/Moscow'), '%C'); -SELECT formatDateTime(toDateTime64('2105-12-12 12:12:12', 0, 'Europe/Moscow'), '%C'); -SELECT formatDateTime(toDateTime64('2205-12-12 12:12:12', 0, 'Europe/Moscow'), '%C'); +SELECT formatDateTime(toDateTime64('1935-12-12 12:12:12', 0, 'Asia/Istanbul'), '%C'); +SELECT formatDateTime(toDateTime64('1969-12-12 12:12:12', 0, 'Asia/Istanbul'), '%C'); +SELECT formatDateTime(toDateTime64('1989-12-12 12:12:12', 0, 'Asia/Istanbul'), '%C'); +SELECT formatDateTime(toDateTime64('2019-09-16 19:20:12', 0, 'Asia/Istanbul'), '%C'); +SELECT formatDateTime(toDateTime64('2105-12-12 12:12:12', 0, 'Asia/Istanbul'), '%C'); +SELECT formatDateTime(toDateTime64('2205-12-12 12:12:12', 0, 'Asia/Istanbul'), '%C'); -- non-zero scale -SELECT formatDateTime(toDateTime64('1935-12-12 12:12:12', 6, 'Europe/Moscow'), '%C'); -SELECT formatDateTime(toDateTime64('1969-12-12 12:12:12', 6, 'Europe/Moscow'), '%C'); -SELECT formatDateTime(toDateTime64('1989-12-12 12:12:12', 6, 'Europe/Moscow'), '%C'); -SELECT formatDateTime(toDateTime64('2019-09-16 19:20:12', 0, 'Europe/Moscow'), '%C'); -SELECT formatDateTime(toDateTime64('2105-12-12 12:12:12', 6, 'Europe/Moscow'), '%C'); -SELECT formatDateTime(toDateTime64('2205-01-12 12:12:12', 6, 'Europe/Moscow'), '%C'); \ No newline at end of file +SELECT formatDateTime(toDateTime64('1935-12-12 12:12:12', 6, 'Asia/Istanbul'), '%C'); +SELECT formatDateTime(toDateTime64('1969-12-12 12:12:12', 6, 'Asia/Istanbul'), '%C'); +SELECT formatDateTime(toDateTime64('1989-12-12 12:12:12', 6, 'Asia/Istanbul'), '%C'); +SELECT formatDateTime(toDateTime64('2019-09-16 19:20:12', 0, 'Asia/Istanbul'), '%C'); +SELECT formatDateTime(toDateTime64('2105-12-12 12:12:12', 6, 'Asia/Istanbul'), '%C'); +SELECT formatDateTime(toDateTime64('2205-01-12 12:12:12', 6, 'Asia/Istanbul'), '%C'); \ No newline at end of file diff --git a/tests/queries/0_stateless/01802_toDateTime64_large_values.reference b/tests/queries/0_stateless/01802_toDateTime64_large_values.reference index c44c61ab93a..e60b1c30314 100644 --- a/tests/queries/0_stateless/01802_toDateTime64_large_values.reference +++ b/tests/queries/0_stateless/01802_toDateTime64_large_values.reference @@ -2,9 +2,9 @@ SELECT toDateTime64('2205-12-12 12:12:12', 0, 'UTC'); 2205-12-12 12:12:12 -SELECT toDateTime64('2205-12-12 12:12:12', 0, 'Europe/Moscow'); +SELECT toDateTime64('2205-12-12 12:12:12', 0, 'Asia/Istanbul'); 2205-12-12 12:12:12 -SELECT toDateTime64('2205-12-12 12:12:12', 6, 'Europe/Moscow'); +SELECT toDateTime64('2205-12-12 12:12:12', 6, 'Asia/Istanbul'); 2205-12-12 12:12:12.000000 -SELECT toDateTime64('2205-12-12 12:12:12', 6, 'Europe/Moscow'); +SELECT toDateTime64('2205-12-12 12:12:12', 6, 'Asia/Istanbul'); 2205-12-12 12:12:12.000000 diff --git a/tests/queries/0_stateless/01802_toDateTime64_large_values.sql b/tests/queries/0_stateless/01802_toDateTime64_large_values.sql index 299111f43bc..d82d4433b2d 100644 --- a/tests/queries/0_stateless/01802_toDateTime64_large_values.sql +++ b/tests/queries/0_stateless/01802_toDateTime64_large_values.sql @@ -1,7 +1,7 @@ -- { echo } SELECT toDateTime64('2205-12-12 12:12:12', 0, 'UTC'); -SELECT toDateTime64('2205-12-12 12:12:12', 0, 'Europe/Moscow'); +SELECT toDateTime64('2205-12-12 12:12:12', 0, 'Asia/Istanbul'); -SELECT toDateTime64('2205-12-12 12:12:12', 6, 'Europe/Moscow'); -SELECT toDateTime64('2205-12-12 12:12:12', 6, 'Europe/Moscow'); \ No newline at end of file +SELECT toDateTime64('2205-12-12 12:12:12', 6, 'Asia/Istanbul'); +SELECT toDateTime64('2205-12-12 12:12:12', 6, 'Asia/Istanbul'); \ No newline at end of file diff --git a/tests/queries/0_stateless/01811_datename.sql b/tests/queries/0_stateless/01811_datename.sql index 0cd538b52c7..b757d9ae018 100644 --- a/tests/queries/0_stateless/01811_datename.sql +++ b/tests/queries/0_stateless/01811_datename.sql @@ -66,7 +66,7 @@ SELECT WITH toDateTime('2021-04-14 23:22:33', 'UTC') as date SELECT - dateName('weekday', date, 'Europe/Moscow'), - dateName('hour', date, 'Europe/Moscow'), - dateName('minute', date, 'Europe/Moscow'), - dateName('second', date, 'Europe/Moscow'); + dateName('weekday', date, 'Asia/Istanbul'), + dateName('hour', date, 'Asia/Istanbul'), + dateName('minute', date, 'Asia/Istanbul'), + dateName('second', date, 'Asia/Istanbul'); diff --git a/tests/queries/0_stateless/01821_to_date_time_ubsan.sql b/tests/queries/0_stateless/01821_to_date_time_ubsan.sql index 377291e015f..5ec767fe413 100644 --- a/tests/queries/0_stateless/01821_to_date_time_ubsan.sql +++ b/tests/queries/0_stateless/01821_to_date_time_ubsan.sql @@ -1,2 +1,2 @@ -SELECT toDateTime('9223372036854775806', 7, 'Europe/Moscow'); -SELECT toDateTime('9223372036854775806', 8, 'Europe/Moscow'); +SELECT toDateTime('9223372036854775806', 7, 'Asia/Istanbul'); +SELECT toDateTime('9223372036854775806', 8, 'Asia/Istanbul'); diff --git a/tests/queries/0_stateless/01852_map_combinator.sql b/tests/queries/0_stateless/01852_map_combinator.sql index 3036e2e0ea4..a23a507bc27 100644 --- a/tests/queries/0_stateless/01852_map_combinator.sql +++ b/tests/queries/0_stateless/01852_map_combinator.sql @@ -26,7 +26,7 @@ select minMap(val) from values ('val Map(String, String)', (map('1', '1')), (ma select minMap(val) from values ('val Map(FixedString(1), FixedString(1))', (map('1', '1')), (map('1', '2'))); select minMap(val) from values ('val Map(UInt64, UInt64)', (map(1, 1)), (map(1, 2))); select minMap(val) from values ('val Map(Date, Int16)', (map(1, 1)), (map(1, 2))); -select minMap(val) from values ('val Map(DateTime(\'Europe/Moscow\'), Int32)', (map(1, 1)), (map(1, 2))); +select minMap(val) from values ('val Map(DateTime(\'Asia/Istanbul\'), Int32)', (map(1, 1)), (map(1, 2))); select minMap(val) from values ('val Map(Enum16(\'a\'=1), Int16)', (map('a', 1)), (map('a', 2))); select maxMap(val) from values ('val Map(String, String)', (map('1', '1')), (map('1', '2'))); select minMap(val) from values ('val Map(Int128, Int128)', (map(1, 1)), (map(1, 2))); diff --git a/tests/queries/0_stateless/01867_support_datetime64_version_column.sql b/tests/queries/0_stateless/01867_support_datetime64_version_column.sql index 1aea0fb91f2..2f0ed1fdc7f 100644 --- a/tests/queries/0_stateless/01867_support_datetime64_version_column.sql +++ b/tests/queries/0_stateless/01867_support_datetime64_version_column.sql @@ -1,5 +1,5 @@ drop table if exists replacing; -create table replacing( `A` Int64, `D` DateTime64(9, 'Europe/Moscow'), `S` String) ENGINE = ReplacingMergeTree(D) ORDER BY A; +create table replacing( `A` Int64, `D` DateTime64(9, 'Asia/Istanbul'), `S` String) ENGINE = ReplacingMergeTree(D) ORDER BY A; insert into replacing values (1,'1970-01-01 08:25:46.300800000','a'); insert into replacing values (2,'1970-01-01 08:25:46.300800002','b'); diff --git a/tests/queries/0_stateless/01868_order_by_fill_with_datetime64.sql b/tests/queries/0_stateless/01868_order_by_fill_with_datetime64.sql index ff3134d37ed..3a49ef73d1a 100644 --- a/tests/queries/0_stateless/01868_order_by_fill_with_datetime64.sql +++ b/tests/queries/0_stateless/01868_order_by_fill_with_datetime64.sql @@ -1,2 +1,2 @@ -SELECT n, source FROM (SELECT toDateTime64(number * 1000, 3,'Europe/Moscow') AS n, 'original' AS source FROM numbers(10) WHERE (number % 3) = 1 ) ORDER BY n ASC WITH FILL STEP toDateTime64(1000, 3); -SELECT n, source FROM (SELECT toDateTime64(number * 1000, 9,'Europe/Moscow') AS n, 'original' AS source FROM numbers(10) WHERE (number % 3) = 1 ) ORDER BY n ASC WITH FILL STEP toDateTime64(1000, 9); +SELECT n, source FROM (SELECT toDateTime64(number * 1000, 3,'Asia/Istanbul') AS n, 'original' AS source FROM numbers(10) WHERE (number % 3) = 1 ) ORDER BY n ASC WITH FILL STEP toDateTime64(1000, 3); +SELECT n, source FROM (SELECT toDateTime64(number * 1000, 9,'Asia/Istanbul') AS n, 'original' AS source FROM numbers(10) WHERE (number % 3) = 1 ) ORDER BY n ASC WITH FILL STEP toDateTime64(1000, 9); diff --git a/tests/queries/0_stateless/01891_partition_hash.sql b/tests/queries/0_stateless/01891_partition_hash.sql index f401c7c2d07..f56ed6a4ff4 100644 --- a/tests/queries/0_stateless/01891_partition_hash.sql +++ b/tests/queries/0_stateless/01891_partition_hash.sql @@ -1,5 +1,5 @@ drop table if exists tab; -create table tab (i8 Int8, i16 Int16, i32 Int32, i64 Int64, i128 Int128, i256 Int256, u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64, u128 UInt128, u256 UInt256, id UUID, s String, fs FixedString(33), a Array(UInt8), t Tuple(UInt16, UInt32), d Date, dt DateTime('Europe/Moscow'), dt64 DateTime64(3, 'Europe/Moscow'), dec128 Decimal128(3), dec256 Decimal256(4), lc LowCardinality(String)) engine = MergeTree PARTITION BY (i8, i16, i32, i64, i128, i256, u8, u16, u32, u64, u128, u256, id, s, fs, a, t, d, dt, dt64, dec128, dec256, lc) order by tuple(); +create table tab (i8 Int8, i16 Int16, i32 Int32, i64 Int64, i128 Int128, i256 Int256, u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64, u128 UInt128, u256 UInt256, id UUID, s String, fs FixedString(33), a Array(UInt8), t Tuple(UInt16, UInt32), d Date, dt DateTime('Asia/Istanbul'), dt64 DateTime64(3, 'Asia/Istanbul'), dec128 Decimal128(3), dec256 Decimal256(4), lc LowCardinality(String)) engine = MergeTree PARTITION BY (i8, i16, i32, i64, i128, i256, u8, u16, u32, u64, u128, u256, id, s, fs, a, t, d, dt, dt64, dec128, dec256, lc) order by tuple(); insert into tab values (-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, '61f0c404-5cb3-11e7-907b-a6006ad3dba0', 'a', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', [1, 2, 3], (-1, -2), '2020-01-01', '2020-01-01 01:01:01', '2020-01-01 01:01:01', '123.456', '78.9101', 'a'); -- Here we check that partition id did not change. -- Different result means Backward Incompatible Change. Old partitions will not be accepted by new server. diff --git a/tests/queries/0_stateless/01891_partition_hash_no_long_int.sql b/tests/queries/0_stateless/01891_partition_hash_no_long_int.sql index 643266f1ea3..431f566b806 100644 --- a/tests/queries/0_stateless/01891_partition_hash_no_long_int.sql +++ b/tests/queries/0_stateless/01891_partition_hash_no_long_int.sql @@ -1,7 +1,7 @@ -- Tags: long drop table if exists tab; -create table tab (i8 Int8, i16 Int16, i32 Int32, i64 Int64, u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64, id UUID, s String, fs FixedString(33), a Array(UInt8), t Tuple(UInt16, UInt32), d Date, dt DateTime('Europe/Moscow'), dt64 DateTime64(3, 'Europe/Moscow'), dec128 Decimal128(3), lc LowCardinality(String)) engine = MergeTree PARTITION BY (i8, i16, i32, i64, u8, u16, u32, u64, id, s, fs, a, t, d, dt, dt64, dec128, lc) order by tuple(); +create table tab (i8 Int8, i16 Int16, i32 Int32, i64 Int64, u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64, id UUID, s String, fs FixedString(33), a Array(UInt8), t Tuple(UInt16, UInt32), d Date, dt DateTime('Asia/Istanbul'), dt64 DateTime64(3, 'Asia/Istanbul'), dec128 Decimal128(3), lc LowCardinality(String)) engine = MergeTree PARTITION BY (i8, i16, i32, i64, u8, u16, u32, u64, id, s, fs, a, t, d, dt, dt64, dec128, lc) order by tuple(); insert into tab values (-1, -1, -1, -1, -1, -1, -1, -1, '61f0c404-5cb3-11e7-907b-a6006ad3dba0', 'a', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', [1, 2, 3], (-1, -2), '2020-01-01', '2020-01-01 01:01:01', '2020-01-01 01:01:01', '123.456', 'a'); -- Here we check that partition id did not change. -- Different result means Backward Incompatible Change. Old partitions will not be accepted by new server. diff --git a/tests/queries/0_stateless/01905_to_json_string.sql b/tests/queries/0_stateless/01905_to_json_string.sql index e92c32f3422..38c02ef13fb 100644 --- a/tests/queries/0_stateless/01905_to_json_string.sql +++ b/tests/queries/0_stateless/01905_to_json_string.sql @@ -8,7 +8,7 @@ $$ d Decimal32(4), e Nullable(Enum16('h' = 1, 'w' = 5 , 'o' = -200)), f Float64, - g Tuple(Date, DateTime('Europe/Moscow'), DateTime64(3, 'Europe/Moscow'), UUID), + g Tuple(Date, DateTime('Asia/Istanbul'), DateTime64(3, 'Asia/Istanbul'), UUID), h FixedString(2), i Array(Nullable(UUID)) $$, 10, 5, 3) limit 2; diff --git a/tests/queries/0_stateless/01921_datatype_date32.sql b/tests/queries/0_stateless/01921_datatype_date32.sql index e01bdfeee8d..49e5366b455 100644 --- a/tests/queries/0_stateless/01921_datatype_date32.sql +++ b/tests/queries/0_stateless/01921_datatype_date32.sql @@ -23,7 +23,7 @@ select toMinute(x1) from t1; -- { serverError 43 } select '-------toSecond---------'; select toSecond(x1) from t1; -- { serverError 43 } select '-------toStartOfDay---------'; -select toStartOfDay(x1, 'Europe/Moscow') from t1; +select toStartOfDay(x1, 'Asia/Istanbul') from t1; select '-------toMonday---------'; select toMonday(x1) from t1; select '-------toISOWeek---------'; @@ -57,21 +57,21 @@ select toStartOfHour(x1) from t1; -- { serverError 43 } select '-------toStartOfISOYear---------'; select toStartOfISOYear(x1) from t1; select '-------toRelativeYearNum---------'; -select toRelativeYearNum(x1, 'Europe/Moscow') from t1; +select toRelativeYearNum(x1, 'Asia/Istanbul') from t1; select '-------toRelativeQuarterNum---------'; -select toRelativeQuarterNum(x1, 'Europe/Moscow') from t1; +select toRelativeQuarterNum(x1, 'Asia/Istanbul') from t1; select '-------toRelativeMonthNum---------'; -select toRelativeMonthNum(x1, 'Europe/Moscow') from t1; +select toRelativeMonthNum(x1, 'Asia/Istanbul') from t1; select '-------toRelativeWeekNum---------'; -select toRelativeWeekNum(x1, 'Europe/Moscow') from t1; +select toRelativeWeekNum(x1, 'Asia/Istanbul') from t1; select '-------toRelativeDayNum---------'; -select toRelativeDayNum(x1, 'Europe/Moscow') from t1; +select toRelativeDayNum(x1, 'Asia/Istanbul') from t1; select '-------toRelativeHourNum---------'; -select toRelativeHourNum(x1, 'Europe/Moscow') from t1; +select toRelativeHourNum(x1, 'Asia/Istanbul') from t1; select '-------toRelativeMinuteNum---------'; -select toRelativeMinuteNum(x1, 'Europe/Moscow') from t1; +select toRelativeMinuteNum(x1, 'Asia/Istanbul') from t1; select '-------toRelativeSecondNum---------'; -select toRelativeSecondNum(x1, 'Europe/Moscow') from t1; +select toRelativeSecondNum(x1, 'Asia/Istanbul') from t1; select '-------toTime---------'; select toTime(x1) from t1; -- { serverError 43 } select '-------toYYYYMM---------'; diff --git a/tests/queries/0_stateless/01925_date_date_time_comparison.sql b/tests/queries/0_stateless/01925_date_date_time_comparison.sql index 13e856384d2..0659d85b028 100644 --- a/tests/queries/0_stateless/01925_date_date_time_comparison.sql +++ b/tests/queries/0_stateless/01925_date_date_time_comparison.sql @@ -1,2 +1,2 @@ -SELECT toDate('2000-01-01') < toDateTime('2000-01-01 00:00:01', 'Europe/Moscow'); -SELECT toDate('2000-01-01') < toDateTime64('2000-01-01 00:00:01', 0, 'Europe/Moscow'); +SELECT toDate('2000-01-01') < toDateTime('2000-01-01 00:00:01', 'Asia/Istanbul'); +SELECT toDate('2000-01-01') < toDateTime64('2000-01-01 00:00:01', 0, 'Asia/Istanbul'); diff --git a/tests/queries/0_stateless/01926_date_date_time_supertype.reference b/tests/queries/0_stateless/01926_date_date_time_supertype.reference index ec9933dfbd2..e4e8ddfceab 100644 --- a/tests/queries/0_stateless/01926_date_date_time_supertype.reference +++ b/tests/queries/0_stateless/01926_date_date_time_supertype.reference @@ -1,12 +1,12 @@ Array -Array(DateTime(\'Europe/Moscow\')) -Array(DateTime64(5, \'Europe/Moscow\')) -Array(DateTime64(6, \'Europe/Moscow\')) +Array(DateTime(\'Asia/Istanbul\')) +Array(DateTime64(5, \'Asia/Istanbul\')) +Array(DateTime64(6, \'Asia/Istanbul\')) If -2000-01-01 00:00:00 DateTime(\'Europe/Moscow\') -2000-01-01 00:00:00 DateTime(\'Europe/Moscow\') -2000-01-01 00:00:00.00000 DateTime64(5, \'Europe/Moscow\') -2000-01-01 00:00:00.00000 DateTime64(5, \'Europe/Moscow\') +2000-01-01 00:00:00 DateTime(\'Asia/Istanbul\') +2000-01-01 00:00:00 DateTime(\'Asia/Istanbul\') +2000-01-01 00:00:00.00000 DateTime64(5, \'Asia/Istanbul\') +2000-01-01 00:00:00.00000 DateTime64(5, \'Asia/Istanbul\') Cast 2000-01-01 00:00:00 DateTime(\'UTC\') 2000-01-01 00:00:00.00000 DateTime64(5, \'UTC\') diff --git a/tests/queries/0_stateless/01926_date_date_time_supertype.sql b/tests/queries/0_stateless/01926_date_date_time_supertype.sql index cce488a5cff..756fd04a01f 100644 --- a/tests/queries/0_stateless/01926_date_date_time_supertype.sql +++ b/tests/queries/0_stateless/01926_date_date_time_supertype.sql @@ -1,8 +1,8 @@ SELECT 'Array'; -SELECT toTypeName([toDate('2000-01-01'), toDateTime('2000-01-01', 'Europe/Moscow')]); -SELECT toTypeName([toDate('2000-01-01'), toDateTime('2000-01-01', 'Europe/Moscow'), toDateTime64('2000-01-01', 5, 'Europe/Moscow')]); -SELECT toTypeName([toDate('2000-01-01'), toDateTime('2000-01-01', 'Europe/Moscow'), toDateTime64('2000-01-01', 5, 'Europe/Moscow'), toDateTime64('2000-01-01', 6, 'Europe/Moscow')]); +SELECT toTypeName([toDate('2000-01-01'), toDateTime('2000-01-01', 'Asia/Istanbul')]); +SELECT toTypeName([toDate('2000-01-01'), toDateTime('2000-01-01', 'Asia/Istanbul'), toDateTime64('2000-01-01', 5, 'Asia/Istanbul')]); +SELECT toTypeName([toDate('2000-01-01'), toDateTime('2000-01-01', 'Asia/Istanbul'), toDateTime64('2000-01-01', 5, 'Asia/Istanbul'), toDateTime64('2000-01-01', 6, 'Asia/Istanbul')]); DROP TABLE IF EXISTS predicate_table; CREATE TABLE predicate_table (value UInt8) ENGINE=TinyLog; @@ -11,11 +11,11 @@ INSERT INTO predicate_table VALUES (0), (1); SELECT 'If'; -WITH toDate('2000-01-01') as a, toDateTime('2000-01-01', 'Europe/Moscow') as b +WITH toDate('2000-01-01') as a, toDateTime('2000-01-01', 'Asia/Istanbul') as b SELECT if(value, b, a) as result, toTypeName(result) FROM predicate_table; -WITH toDateTime('2000-01-01', 'Europe/Moscow') as a, toDateTime64('2000-01-01', 5, 'Europe/Moscow') as b +WITH toDateTime('2000-01-01', 'Asia/Istanbul') as a, toDateTime64('2000-01-01', 5, 'Asia/Istanbul') as b SELECT if(value, b, a) as result, toTypeName(result) FROM predicate_table; diff --git a/tests/queries/0_stateless/02041_conversion_between_date32_and_datetime64.sql b/tests/queries/0_stateless/02041_conversion_between_date32_and_datetime64.sql index 9a25f2b007b..05e5a090d86 100644 --- a/tests/queries/0_stateless/02041_conversion_between_date32_and_datetime64.sql +++ b/tests/queries/0_stateless/02041_conversion_between_date32_and_datetime64.sql @@ -1 +1 @@ -select toDate32(toDateTime64('2019-01-01 00:00:00', 3, 'Europe/Moscow')), toDateTime64(toDate32('2019-01-01'), 3, 'Europe/Moscow') \ No newline at end of file +select toDate32(toDateTime64('2019-01-01 00:00:00', 3, 'Asia/Istanbul')), toDateTime64(toDate32('2019-01-01'), 3, 'Asia/Istanbul') \ No newline at end of file diff --git a/tests/queries/0_stateless/02096_date_time_1970_saturation.sql b/tests/queries/0_stateless/02096_date_time_1970_saturation.sql index e0c401443a7..8cd472c4e6c 100644 --- a/tests/queries/0_stateless/02096_date_time_1970_saturation.sql +++ b/tests/queries/0_stateless/02096_date_time_1970_saturation.sql @@ -1,21 +1,21 @@ select toDate(0); -select toDateTime(0, 'Europe/Moscow'); +select toDateTime(0, 'Asia/Istanbul'); select toMonday(toDate(0)); -select toMonday(toDateTime(0, 'Europe/Moscow')); +select toMonday(toDateTime(0, 'Asia/Istanbul')); select toStartOfWeek(toDate(0)); -select toStartOfWeek(toDateTime(0, 'Europe/Moscow')); +select toStartOfWeek(toDateTime(0, 'Asia/Istanbul')); select toStartOfMonth(toDate(0)); -select toStartOfMonth(toDateTime(0, 'Europe/Moscow')); +select toStartOfMonth(toDateTime(0, 'Asia/Istanbul')); select toStartOfQuarter(toDate(0)); -select toStartOfQuarter(toDateTime(0, 'Europe/Moscow')); +select toStartOfQuarter(toDateTime(0, 'Asia/Istanbul')); select toStartOfYear(toDate(0)); -select toStartOfYear(toDateTime(0, 'Europe/Moscow')); -select toTime(toDateTime(0, 'Europe/Moscow')); -select toStartOfMinute(toDateTime(0, 'Europe/Moscow')); -select toStartOfFiveMinute(toDateTime(0, 'Europe/Moscow')); -select toStartOfTenMinutes(toDateTime(0, 'Europe/Moscow')); -select toStartOfFifteenMinutes(toDateTime(0, 'Europe/Moscow')); -select toStartOfHour(toDateTime(0, 'Europe/Moscow')); +select toStartOfYear(toDateTime(0, 'Asia/Istanbul')); +select toTime(toDateTime(0, 'Asia/Istanbul')); +select toStartOfMinute(toDateTime(0, 'Asia/Istanbul')); +select toStartOfFiveMinute(toDateTime(0, 'Asia/Istanbul')); +select toStartOfTenMinutes(toDateTime(0, 'Asia/Istanbul')); +select toStartOfFifteenMinutes(toDateTime(0, 'Asia/Istanbul')); +select toStartOfHour(toDateTime(0, 'Asia/Istanbul')); select toDateTime(0, 'America/Los_Angeles'); select toMonday(toDateTime(0, 'America/Los_Angeles')); diff --git a/tests/queries/0_stateless/02176_toStartOfWeek_overflow_pruning.sql b/tests/queries/0_stateless/02176_toStartOfWeek_overflow_pruning.sql index 7f62e187241..df5499df32a 100644 --- a/tests/queries/0_stateless/02176_toStartOfWeek_overflow_pruning.sql +++ b/tests/queries/0_stateless/02176_toStartOfWeek_overflow_pruning.sql @@ -1,5 +1,5 @@ SELECT toStartOfWeek(toDateTime('1970-01-01 00:00:00', 'UTC')); -SELECT toStartOfWeek(toDateTime('1970-01-01 00:00:00', 'Europe/Moscow')); +SELECT toStartOfWeek(toDateTime('1970-01-01 00:00:00', 'Asia/Istanbul')); SELECT toStartOfWeek(toDateTime('1970-01-01 00:00:00', 'Canada/Atlantic')); SELECT toStartOfWeek(toDateTime('1970-01-04 00:00:00')); diff --git a/tests/queries/0_stateless/02184_default_table_engine.sql b/tests/queries/0_stateless/02184_default_table_engine.sql index d129ccc801e..c0463343956 100644 --- a/tests/queries/0_stateless/02184_default_table_engine.sql +++ b/tests/queries/0_stateless/02184_default_table_engine.sql @@ -38,9 +38,9 @@ SELECT sum(number) FROM numbers3; SHOW CREATE TABLE numbers3; DROP TABLE numbers3; -CREATE TABLE test_table (EventDate Date, CounterID UInt32, UserID UInt64, EventTime DateTime('Europe/Moscow'), UTCEventTime DateTime('UTC')) PARTITION BY EventDate PRIMARY KEY CounterID; +CREATE TABLE test_table (EventDate Date, CounterID UInt32, UserID UInt64, EventTime DateTime('Asia/Istanbul'), UTCEventTime DateTime('UTC')) PARTITION BY EventDate PRIMARY KEY CounterID; SET default_table_engine = 'Memory'; -CREATE MATERIALIZED VIEW test_view (Rows UInt64, MaxHitTime DateTime('Europe/Moscow')) AS SELECT count() AS Rows, max(UTCEventTime) AS MaxHitTime FROM test_table; +CREATE MATERIALIZED VIEW test_view (Rows UInt64, MaxHitTime DateTime('Asia/Istanbul')) AS SELECT count() AS Rows, max(UTCEventTime) AS MaxHitTime FROM test_table; CREATE MATERIALIZED VIEW test_view_filtered (EventDate Date, CounterID UInt32) POPULATE AS SELECT CounterID, EventDate FROM test_table WHERE EventDate < '2013-01-01'; SHOW CREATE TABLE test_view_filtered; INSERT INTO test_table (EventDate, UTCEventTime) VALUES ('2014-01-02', '2014-01-02 03:04:06'); diff --git a/tests/queries/0_stateless/data_parquet/alltypes_list.parquet.columns b/tests/queries/0_stateless/data_parquet/alltypes_list.parquet.columns index 794ee47d757..3bf762ed7d5 100644 --- a/tests/queries/0_stateless/data_parquet/alltypes_list.parquet.columns +++ b/tests/queries/0_stateless/data_parquet/alltypes_list.parquet.columns @@ -1 +1 @@ -`a1` Array(Int8), `a2` Array(UInt8), `a3` Array(Int16), `a4` Array(UInt16), `a5` Array(Int32), `a6` Array(UInt32), `a7` Array(Int64), `a8` Array(UInt64), `a9` Array(String), `a10` Array(FixedString(4)), `a11` Array(Float32), `a12` Array(Float64), `a13` Array(Date), `a14` Array(Datetime('Europe/Moscow')), `a15` Array(Decimal(4, 2)), `a16` Array(Decimal(10, 2)), `a17` Array(Decimal(25, 2)) +`a1` Array(Int8), `a2` Array(UInt8), `a3` Array(Int16), `a4` Array(UInt16), `a5` Array(Int32), `a6` Array(UInt32), `a7` Array(Int64), `a8` Array(UInt64), `a9` Array(String), `a10` Array(FixedString(4)), `a11` Array(Float32), `a12` Array(Float64), `a13` Array(Date), `a14` Array(Datetime('Asia/Istanbul')), `a15` Array(Decimal(4, 2)), `a16` Array(Decimal(10, 2)), `a17` Array(Decimal(25, 2)) diff --git a/tests/queries/0_stateless/data_parquet/v0.7.1.column-metadata-handling.parquet.columns b/tests/queries/0_stateless/data_parquet/v0.7.1.column-metadata-handling.parquet.columns index df35127ede8..c6d754f04c7 100644 --- a/tests/queries/0_stateless/data_parquet/v0.7.1.column-metadata-handling.parquet.columns +++ b/tests/queries/0_stateless/data_parquet/v0.7.1.column-metadata-handling.parquet.columns @@ -1 +1 @@ -`a` Nullable(Int64), `b` Nullable(Float64), `c` Nullable(DateTime('Europe/Moscow')), `index` Nullable(String), `__index_level_1__` Nullable(DateTime('Europe/Moscow')) +`a` Nullable(Int64), `b` Nullable(Float64), `c` Nullable(DateTime('Asia/Istanbul')), `index` Nullable(String), `__index_level_1__` Nullable(DateTime('Asia/Istanbul')) diff --git a/tests/queries/0_stateless/helpers/00900_parquet_create_table_columns.py b/tests/queries/0_stateless/helpers/00900_parquet_create_table_columns.py index 92606c9cb26..a1ce8ed7e65 100755 --- a/tests/queries/0_stateless/helpers/00900_parquet_create_table_columns.py +++ b/tests/queries/0_stateless/helpers/00900_parquet_create_table_columns.py @@ -4,8 +4,8 @@ import json import sys TYPE_PARQUET_CONVERTED_TO_CLICKHOUSE = { - "TIMESTAMP_MICROS": "DateTime('Europe/Moscow')", - "TIMESTAMP_MILLIS": "DateTime('Europe/Moscow')", + "TIMESTAMP_MICROS": "DateTime('Asia/Istanbul')", + "TIMESTAMP_MILLIS": "DateTime('Asia/Istanbul')", "UTF8": "String", } From 26b905be65ab81acdc443dcaedc6764f0d1719ef Mon Sep 17 00:00:00 2001 From: zvonand Date: Sun, 20 Mar 2022 15:38:11 +0300 Subject: [PATCH 0004/1954] old work upload --- src/Functions/nonNegativeDerivative.cpp | 13 + src/Functions/nonNegativeDerivative.h | 234 ++++++++++++++++++ .../registerFunctionsMiscellaneous.cpp | 2 + 3 files changed, 249 insertions(+) create mode 100644 src/Functions/nonNegativeDerivative.cpp create mode 100644 src/Functions/nonNegativeDerivative.h diff --git a/src/Functions/nonNegativeDerivative.cpp b/src/Functions/nonNegativeDerivative.cpp new file mode 100644 index 00000000000..7a68aa5b2fe --- /dev/null +++ b/src/Functions/nonNegativeDerivative.cpp @@ -0,0 +1,13 @@ +#include +#include + + +namespace DB +{ + +void registerFunctionNonNegativeDerivative(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/nonNegativeDerivative.h b/src/Functions/nonNegativeDerivative.h new file mode 100644 index 00000000000..8b58b2a232f --- /dev/null +++ b/src/Functions/nonNegativeDerivative.h @@ -0,0 +1,234 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +/** Calculate derivative of given value column by specified timestamp. */ +class FunctionNonNegativeDerivativeImpl : public IFunction +{ +private: + static NO_SANITIZE_UNDEFINED Int64 getResultScaling() + { + Int64 res = 1; + + return res; + } + + template + static NO_SANITIZE_UNDEFINED void process(const PaddedPODArray & metric, PaddedPODArray & result, + const PaddedPODArray & timestamp, const UInt32 ts_scale, + const std::tuple interval, const NullMap * null_map) + { + size_t size = metric.size(); + result.resize(size); + + if (size == 0) + return; + + Src prev_metric_value{}; + DateTime64 prev_ts_value{}; + + bool first_row = false; + + for (size_t i = 0; i < size; ++i) + { + if (null_map && (*null_map)[i]) + { + result[i] = Dst{}; + continue; + } + + if (!first_row) + { + auto cur = metric[i]; + /// Overflow is Ok. + result[i] = static_cast(cur) - prev_metric_value; + prev_metric_value = cur; + } + else + { + result[i] = 0; + prev_metric_value = metric[i]; + first_row = false; + } + } + } + + /// Result type is same as result of subtraction of argument types. + template + using DstFieldType = typename NumberTraits::ResultOfSubtraction::Type; + + /// Call polymorphic lambda with tag argument of concrete field type of src_type. + template + void dispatchForSourceType(const IDataType & src_type, F && f) const + { + WhichDataType which(src_type); + + if (which.isUInt8()) + f(UInt8()); + else if (which.isUInt16()) + f(UInt16()); + else if (which.isUInt32()) + f(UInt32()); + else if (which.isUInt64()) + f(UInt64()); + else if (which.isInt8()) + f(Int8()); + else if (which.isInt16()) + f(Int16()); + else if (which.isInt32()) + f(Int32()); + else if (which.isInt64()) + f(Int64()); + else if (which.isFloat32()) + f(Float32()); + else if (which.isFloat64()) + f(Float64()); + else if (which.isDate()) + f(DataTypeDate::FieldType()); + else if (which.isDate32()) + f(DataTypeDate::FieldType()); + else if (which.isDateTime()) + f(DataTypeDateTime::FieldType()); + else + throw Exception("First argument for function " + getName() + " must have numeric type.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + +public: + static constexpr auto name = "nonNegativeDerivative"; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool isStateful() const override + { + return true; + } + + size_t getNumberOfArguments() const override + { + return 0; + } + + bool isDeterministic() const override { return false; } + bool isDeterministicInScopeOfQuery() const override + { + return true; + } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + bool useDefaultImplementationForNulls() const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + DataTypePtr res; + dispatchForSourceType(*removeNullable(arguments[0]), [&](auto field_type_tag) + { + res = std::make_shared>>(); + }); + + if (arguments[0]->isNullable()) + res = makeNullable(res); + + return res; + } + + static std::tuple + dispatchForIntervalColumns(const ColumnWithTypeAndName & interval_column) + { + const auto * interval_type = checkAndGetDataType(interval_column.type.get()); + if (!interval_type) + throw Exception("Illegal value" + interval_column.name + "for function nonNegativeDerivative, INTERVAL expected", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + const auto * interval_column_const_int64 = checkAndGetColumnConst(interval_column.column.get()); + if (!interval_column_const_int64) + throw Exception("Illegal value " + interval_column.name + "for function nonNegativeDerivative, INTERVAL expected", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + Int64 num_units = interval_column_const_int64->getValue(); + if (num_units <= 0) + throw Exception("Value for column " + interval_column.name + "for function nonNegativeDerivative must be positive", + ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + return {interval_type->getKind(), num_units}; + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + if (arguments.size() < 2 || arguments.size() > 3) + throw Exception("Invalid number of arguments, expected 2 or 3: nonNegativeDerivative(metric, timestamp[, INTERVAL x SECOND])", + DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const auto & metric = arguments.at(0); + const auto & timestamp = arguments.at(1); + + const auto timestamp_scale = assert_cast(*arguments[0].type).getScale(); + + // Default interval value: INTERVAL 1 SECOND + const auto interval_params = arguments.size() == 3 ? dispatchForIntervalColumns(arguments.at(2)) : std::tuple(IntervalKind::Second, 1); + + /// When column is constant, its derivative is 0. + if (isColumnConst(*metric.column)) + return result_type->createColumnConstWithDefaultValue(input_rows_count); + + auto res_column = removeNullable(result_type)->createColumn(); + + const auto * metric_column = metric.column.get(); + const auto * timestamp_column = timestamp.column.get(); + + ColumnPtr null_map_column = nullptr; + const NullMap * null_map = nullptr; + if (const auto * nullable_column = checkAndGetColumn(metric_column)) + { + metric_column = &nullable_column->getNestedColumn(); + null_map_column = nullable_column->getNullMapColumnPtr(); + null_map = &nullable_column->getNullMapData(); + } + + dispatchForSourceType(*removeNullable(metric.type), [&](auto field_type_tag) + { + using MetricFieldType = decltype(field_type_tag); + + process(assert_cast &>(*metric_column).getData(), + assert_cast> &>(*res_column).getData(), + assert_cast &>(*timestamp_column).getData(), + timestamp_scale, interval_params, null_map); + }); + + if (null_map_column) + return ColumnNullable::create(std::move(res_column), null_map_column); + else + return res_column; + } +}; + +} diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 76d61ce509a..14474b715de 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -80,6 +80,7 @@ void registerFunctionInitialQueryID(FunctionFactory & factory); void registerFunctionServerUUID(FunctionFactory &); void registerFunctionZooKeeperSessionUptime(FunctionFactory &); void registerFunctionGetOSKernelVersion(FunctionFactory &); +void registerFunctionNonNegativeDerivative(FunctionFactory &); #if USE_ICU void registerFunctionConvertCharset(FunctionFactory &); @@ -166,6 +167,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionServerUUID(factory); registerFunctionZooKeeperSessionUptime(factory); registerFunctionGetOSKernelVersion(factory); + registerFunctionNonNegativeDerivative(factory); #if USE_ICU registerFunctionConvertCharset(factory); From 2edafbe7852744e84cb442cd3de5c24843bf622d Mon Sep 17 00:00:00 2001 From: zvonand Date: Tue, 22 Mar 2022 16:05:58 +0300 Subject: [PATCH 0005/1954] update nonNegDer --- src/Functions/nonNegativeDerivative.h | 68 ++++++++++++++++----------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/src/Functions/nonNegativeDerivative.h b/src/Functions/nonNegativeDerivative.h index 8b58b2a232f..1be7e8368b4 100644 --- a/src/Functions/nonNegativeDerivative.h +++ b/src/Functions/nonNegativeDerivative.h @@ -27,16 +27,33 @@ namespace ErrorCodes class FunctionNonNegativeDerivativeImpl : public IFunction { private: - static NO_SANITIZE_UNDEFINED Int64 getResultScaling() + /** Get interval length in seconds **/ + static NO_SANITIZE_UNDEFINED Float64 getResultScaling(const std::tuple interval) { - Int64 res = 1; + auto interval_kind = std::get<0>(interval); + auto interval_length = std::get<1>(interval); - return res; + switch (interval_kind) + { + case IntervalKind::Week: + return interval_length * 604800; + case IntervalKind::Day: + return interval_length * 86400; + case IntervalKind::Hour: + return interval_length * 3600; + case IntervalKind::Minute: + return interval_length * 60; + case IntervalKind::Second: + return interval_length; + default: + throw Exception(fmt::format("Interval kind {}: interval length is variadic, only precise intervals accepted", + IntervalKind(interval_kind).toKeyword()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } } - template + template static NO_SANITIZE_UNDEFINED void process(const PaddedPODArray & metric, PaddedPODArray & result, - const PaddedPODArray & timestamp, const UInt32 ts_scale, + const PaddedPODArray & timestamp, const UInt32 ts_scale, const std::tuple interval, const NullMap * null_map) { size_t size = metric.size(); @@ -46,9 +63,11 @@ private: return; Src prev_metric_value{}; - DateTime64 prev_ts_value{}; + Ts prev_ts{}; - bool first_row = false; + bool first_row = true; + auto interval_length = getResultScaling(interval); + auto ts_scale_multiplier = common::exp10_i64(ts_scale); for (size_t i = 0; i < size; ++i) { @@ -58,25 +77,25 @@ private: continue; } - if (!first_row) - { - auto cur = metric[i]; - /// Overflow is Ok. - result[i] = static_cast(cur) - prev_metric_value; - prev_metric_value = cur; - } - else + if (first_row) { result[i] = 0; prev_metric_value = metric[i]; first_row = false; } + else + { + auto cur = metric[i]; + auto multiply = interval_length * ts_scale_multiplier / (timestamp[i].value - prev_ts.value); + result[i] = (cur - prev_metric_value) * multiply; + prev_metric_value = cur; + } } } /// Result type is same as result of subtraction of argument types. template - using DstFieldType = typename NumberTraits::ResultOfSubtraction::Type; + using DstFieldType = typename NumberTraits::ResultOfFloatingPointDivision::Type; /// Call polymorphic lambda with tag argument of concrete field type of src_type. template @@ -132,16 +151,11 @@ public: return true; } - size_t getNumberOfArguments() const override - { - return 0; - } + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } bool isDeterministic() const override { return false; } - bool isDeterministicInScopeOfQuery() const override - { - return true; - } + bool isDeterministicInScopeOfQuery() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } @@ -191,7 +205,7 @@ public: const auto & metric = arguments.at(0); const auto & timestamp = arguments.at(1); - const auto timestamp_scale = assert_cast(*arguments[0].type).getScale(); + const auto timestamp_scale = assert_cast(*arguments[1].type).getScale(); // Default interval value: INTERVAL 1 SECOND const auto interval_params = arguments.size() == 3 ? dispatchForIntervalColumns(arguments.at(2)) : std::tuple(IntervalKind::Second, 1); @@ -220,8 +234,8 @@ public: process(assert_cast &>(*metric_column).getData(), assert_cast> &>(*res_column).getData(), - assert_cast &>(*timestamp_column).getData(), - timestamp_scale, interval_params, null_map); + assert_cast &>(*timestamp_column).getData(), timestamp_scale, + interval_params, null_map); }); if (null_map_column) From 3e42072d95d1a07a2039cdd90b3808c527f93f08 Mon Sep 17 00:00:00 2001 From: zvonand Date: Tue, 22 Mar 2022 18:09:36 +0300 Subject: [PATCH 0006/1954] fix --- src/Functions/nonNegativeDerivative.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Functions/nonNegativeDerivative.h b/src/Functions/nonNegativeDerivative.h index 1be7e8368b4..71949c7c991 100644 --- a/src/Functions/nonNegativeDerivative.h +++ b/src/Functions/nonNegativeDerivative.h @@ -28,7 +28,7 @@ class FunctionNonNegativeDerivativeImpl : public IFunction { private: /** Get interval length in seconds **/ - static NO_SANITIZE_UNDEFINED Float64 getResultScaling(const std::tuple interval) + static NO_SANITIZE_UNDEFINED Float64 getIntervalLength(const std::tuple interval) { auto interval_kind = std::get<0>(interval); auto interval_length = std::get<1>(interval); @@ -66,7 +66,7 @@ private: Ts prev_ts{}; bool first_row = true; - auto interval_length = getResultScaling(interval); + auto interval_length = getIntervalLength(interval); auto ts_scale_multiplier = common::exp10_i64(ts_scale); for (size_t i = 0; i < size; ++i) @@ -81,6 +81,7 @@ private: { result[i] = 0; prev_metric_value = metric[i]; + prev_ts = timestamp[i]; first_row = false; } else @@ -89,6 +90,7 @@ private: auto multiply = interval_length * ts_scale_multiplier / (timestamp[i].value - prev_ts.value); result[i] = (cur - prev_metric_value) * multiply; prev_metric_value = cur; + prev_ts = timestamp[i]; } } } From 66c026d0ac4fffa3171de008b411a83639140f25 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 25 Feb 2022 20:44:13 +0300 Subject: [PATCH 0007/1954] Fix collision of S3 operation log revision --- src/Disks/DiskDecorator.h | 3 + src/Disks/IDisk.h | 8 + src/Disks/S3/DiskS3.cpp | 52 ++++- src/Disks/S3/DiskS3.h | 16 +- src/Disks/S3/registerDiskS3.cpp | 4 +- src/Storages/MergeTree/DataPartsExchange.cpp | 21 ++ .../__init__.py | 0 .../configs/config.d/clusters.xml | 40 ++++ .../configs/config.d/storage_conf.xml | 35 ++++ .../config.d/storage_conf_another_bucket.xml | 34 +++ .../storage_conf_without_zero_copy.xml | 35 ++++ .../test.py | 195 ++++++++++++++++++ 12 files changed, 435 insertions(+), 8 deletions(-) create mode 100644 tests/integration/test_replicated_merge_tree_s3_restore/__init__.py create mode 100644 tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/clusters.xml create mode 100644 tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf.xml create mode 100644 tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml create mode 100644 tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_without_zero_copy.xml create mode 100644 tests/integration/test_replicated_merge_tree_s3_restore/test.py diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index bace54ff22a..32ebde3ac43 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -78,6 +78,9 @@ public: UInt32 getRefCount(const String & path) const override { return delegate->getRefCount(path); } + void syncRevision(UInt64 revision) override { delegate->syncRevision(revision); } + UInt64 getRevision() const override { return delegate->getRevision(); } + protected: Executor & getExecutor() override; diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 5068ac5dde9..16bb111b1eb 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -304,6 +304,14 @@ public: /// other alive harlinks will not be removed. virtual UInt32 getRefCount(const String &) const { return 0; } + /// Revision is an incremental counter of disk operaion. + /// Revision currently exisis only in DiskS3. + /// It is used to save current state during backup and restore that state from backup. + /// This methos sets current disk revision if it lower than required. + virtual void syncRevision(UInt64) {} + /// Return current disk revision. + virtual UInt64 getRevision() const { return 0; } + protected: friend class DiskDecorator; diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index b7e752edafd..87c95a91deb 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -155,12 +155,14 @@ DiskS3::DiskS3( DiskPtr metadata_disk_, ContextPtr context_, SettingsPtr settings_, - GetDiskSettings settings_getter_) + GetDiskSettings settings_getter_, + String operation_log_suffix_) : IDiskRemote(name_, s3_root_path_, metadata_disk_, "DiskS3", settings_->thread_pool_size) , bucket(std::move(bucket_)) , current_settings(std::move(settings_)) , settings_getter(settings_getter_) , context(context_) + , operation_log_suffix(operation_log_suffix_) { } @@ -331,7 +333,7 @@ void DiskS3::shutdown() void DiskS3::createFileOperationObject(const String & operation_name, UInt64 revision, const DiskS3::ObjectMetadata & metadata) { auto settings = current_settings.get(); - const String key = "operations/r" + revisionToString(revision) + "-" + operation_name; + const String key = "operations/r" + revisionToString(revision) + operation_log_suffix + "-" + operation_name; WriteBufferFromS3 buffer( settings->client, bucket, @@ -909,6 +911,36 @@ void DiskS3::processRestoreFiles(const String & source_bucket, const String & so } } +void DiskS3::moveRecursiveOrRemove(const String & from_path, const String & to_path, bool send_metadata) +{ + if (exists(to_path)) + { + if (send_metadata) + { + auto revision = ++revision_counter; + const ObjectMetadata object_metadata { + {"from_path", from_path}, + {"to_path", to_path} + }; + createFileOperationObject("rename", revision, object_metadata); + } + if (isDirectory(from_path)) + { + for (auto it = iterateDirectory(from_path); it->isValid(); it->next()) + moveRecursiveOrRemove(it->path(), fs::path(to_path) / it->name(), false); + } + else + { + removeFile(from_path); + LOG_WARNING(log, "Collision in S3 operation log: rename from '{}' to '{}', file removed", from_path, to_path); + } + } + else + { + moveFile(from_path, to_path, send_metadata); + } +} + void DiskS3::restoreFileOperations(const RestoreInformation & restore_information) { auto settings = current_settings.get(); @@ -951,7 +983,7 @@ void DiskS3::restoreFileOperations(const RestoreInformation & restore_informatio auto to_path = object_metadata["to_path"]; if (exists(from_path)) { - moveFile(from_path, to_path, send_metadata); + moveRecursiveOrRemove(from_path, to_path, send_metadata); LOG_TRACE(log, "Revision {}. Restored rename {} -> {}", revision, from_path, to_path); if (restore_information.detached && isDirectory(to_path)) @@ -1034,9 +1066,10 @@ void DiskS3::restoreFileOperations(const RestoreInformation & restore_informatio std::tuple DiskS3::extractRevisionAndOperationFromKey(const String & key) { String revision_str; + String suffix; String operation; - re2::RE2::FullMatch(key, key_regexp, &revision_str, &operation); + re2::RE2::FullMatch(key, key_regexp, &revision_str, &suffix, &operation); return {(revision_str.empty() ? UNKNOWN_REVISION : static_cast(std::bitset<64>(revision_str).to_ullong())), operation}; } @@ -1079,6 +1112,17 @@ void DiskS3::applyNewSettings(const Poco::Util::AbstractConfiguration & config, exec->setMaxThreads(current_settings.get()->thread_pool_size); } +void DiskS3::syncRevision(UInt64 revision) +{ + UInt64 local_revision = revision_counter.load(); + while ((revision > local_revision) && revision_counter.compare_exchange_weak(local_revision, revision)); +} + +UInt64 DiskS3::getRevision() const +{ + return revision_counter.load(); +} + DiskS3Settings::DiskS3Settings( const std::shared_ptr & client_, size_t s3_max_single_read_retries_, diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index 698fa6173c2..8095b281641 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -75,7 +75,8 @@ public: DiskPtr metadata_disk_, ContextPtr context_, SettingsPtr settings_, - GetDiskSettings settings_getter_); + GetDiskSettings settings_getter_, + String operation_log_suffix_); std::unique_ptr readFile( const String & path, @@ -117,6 +118,9 @@ public: void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String &, const DisksMap &) override; + void syncRevision(UInt64 revision) override; + UInt64 getRevision() const override; + private: void createFileOperationObject(const String & operation_name, UInt64 revision, const ObjectMetadata & metadata); /// Converts revision to binary string with leading zeroes (64 bit). @@ -159,6 +163,10 @@ private: /// Forms detached path '../../detached/part_name/' from '../../part_name/' static String pathToDetached(const String & source_path); + /// Move file or files in directory when possible and remove files in other case + /// to restore by S3 operation log with same operations from different replicas + void moveRecursiveOrRemove(const String & from_path, const String & to_path, bool send_metadata); + const String bucket; MultiVersion current_settings; @@ -172,8 +180,8 @@ private: /// File at path {metadata_path}/restore contains metadata restore information inline static const String RESTORE_FILE_NAME = "restore"; - /// Key has format: ../../r{revision}-{operation} - const re2::RE2 key_regexp {".*/r(\\d+)-(\\w+)$"}; + /// Key has format: ../../r{revision}(-{hostname})-{operation} + const re2::RE2 key_regexp {".*/r(\\d+)(-[\\w\\d\\-\\.]+)?-(\\w+)$"}; /// Object contains information about schema version. inline static const String SCHEMA_VERSION_OBJECT = ".SCHEMA_VERSION"; @@ -183,6 +191,8 @@ private: const std::vector data_roots {"data", "store"}; ContextPtr context; + + String operation_log_suffix; }; } diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index 9b2e7137d53..9f2fc5dd5d1 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -9,6 +9,7 @@ #if USE_AWS_S3 #include +#include #include #include "DiskS3.h" #include "Disks/DiskCacheWrapper.h" @@ -192,7 +193,8 @@ void registerDiskS3(DiskFactory & factory) metadata_disk, context, getSettings(config, config_prefix, context), - getSettings); + getSettings, + "-" + getFQDNOrHostName()); /// This code is used only to check access to the corresponding disk. if (!config.getBool(config_prefix + ".skip_access_check", false)) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 19d990d7c2d..b22aa91af30 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -141,6 +141,16 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedSend}; + { + auto disk = part->volume->getDisk(); + UInt64 revision = parse(params.get("disk_revision", "0")); + if (revision) + disk->syncRevision(revision); + revision = disk->getRevision(); + if (revision) + response.addCookie({"disk_revision", toString(revision)}); + } + if (client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE) writeBinary(part->checksums.getTotalSizeOnDisk(), out); @@ -419,6 +429,13 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( {"compress", "false"} }); + if (disk) + { + UInt64 revision = disk->getRevision(); + if (revision) + uri.addQueryParameter("disk_revision", toString(revision)); + } + Strings capability; if (try_zero_copy && data_settings->allow_remote_fs_zero_copy_replication) { @@ -503,6 +520,10 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( if (!disk) disk = reservation->getDisk(); + UInt64 revision = parse(in.getResponseCookie("disk_revision", "0")); + if (revision) + disk->syncRevision(revision); + bool sync = (data_settings->min_compressed_bytes_to_fsync_after_fetch && sum_files_size >= data_settings->min_compressed_bytes_to_fsync_after_fetch); diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/__init__.py b/tests/integration/test_replicated_merge_tree_s3_restore/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/clusters.xml b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/clusters.xml new file mode 100644 index 00000000000..31f7e06db3c --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/clusters.xml @@ -0,0 +1,40 @@ + + + + + + true + + node1z + 9000 + + + node2z + 9000 + + + + + + true + + node1n + 9000 + + + node2n + 9000 + + + + + + true + + node_another_bucket + 9000 + + + + + diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf.xml new file mode 100644 index 00000000000..63fe7ac1769 --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf.xml @@ -0,0 +1,35 @@ + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + true + 1 + + + local + / + + + + + +
+ s3 +
+ + hdd + +
+
+
+
+ + + 0 + 1 + +
diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml new file mode 100644 index 00000000000..920db2c9edd --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_another_bucket.xml @@ -0,0 +1,34 @@ + + + + + s3 + http://minio1:9001/root2/data/ + minio + minio123 + true + 1 + + + local + / + + + + + +
+ s3 +
+ + hdd + +
+
+
+
+ + + 0 + +
diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_without_zero_copy.xml b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_without_zero_copy.xml new file mode 100644 index 00000000000..9194d779d16 --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3_restore/configs/config.d/storage_conf_without_zero_copy.xml @@ -0,0 +1,35 @@ + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + true + 1 + + + local + / + + + + + +
+ s3 +
+ + hdd + +
+
+
+
+ + + 0 + 0 + +
diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/test.py b/tests/integration/test_replicated_merge_tree_s3_restore/test.py new file mode 100644 index 00000000000..fbf595644f7 --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3_restore/test.py @@ -0,0 +1,195 @@ +import os +import logging +import random +import string +import time + +import pytest +from helpers.cluster import ClickHouseCluster, get_instances_dir + + +COMMON_CONFIGS = ["configs/config.d/clusters.xml"] + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + + cluster.add_instance("node1z", + main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf.xml"], + macros={"cluster": "node_zero_copy", "replica": "0"}, + with_minio=True, with_zookeeper=True, stay_alive=True) + cluster.add_instance("node2z", + main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf.xml"], + macros={"cluster": "node_zero_copy", "replica": "1"}, + with_zookeeper=True, stay_alive=True) + cluster.add_instance("node1n", + main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf_without_zero_copy.xml"], + macros={"cluster": "node_no_zero_copy", "replica": "2"}, + with_minio=True, with_zookeeper=True, stay_alive=True) + cluster.add_instance("node2n", + main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf_without_zero_copy.xml"], + macros={"cluster": "node_no_zero_copy", "replica": "3"}, + with_zookeeper=True, stay_alive=True) + cluster.add_instance("node_another_bucket", + main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf_another_bucket.xml"], + macros={"cluster": "node_another_bucket", "replica": "0"}, + with_zookeeper=True, stay_alive=True) + + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + yield cluster + finally: + cluster.shutdown() + + +def random_string(length): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(length)) + + +def create_table(node, table_name, schema, attach=False, db_atomic=False, uuid=""): + node.query("CREATE DATABASE IF NOT EXISTS s3 {on_cluster} ENGINE = {engine}".format(engine="Atomic" if db_atomic else "Ordinary", + on_cluster="ON CLUSTER '{cluster}'")) + + create_table_statement = """ + {create} TABLE s3.{table_name} {uuid} {on_cluster} ( + key UInt32, + {schema} + ) ENGINE={engine} + PARTITION BY key + ORDER BY key + SETTINGS + storage_policy='s3', + old_parts_lifetime=600, + index_granularity=512 + """.format(create="ATTACH" if attach else "CREATE", + table_name=table_name, + uuid="UUID '{uuid}'".format(uuid=uuid) if db_atomic and uuid else "", + on_cluster="ON CLUSTER '{cluster}'", + schema=schema, + engine="ReplicatedMergeTree('/clickhouse/tables/{cluster}/test', '{replica}')") + + node.query(create_table_statement) + + +def purge_s3(cluster, bucket): + minio = cluster.minio_client + for obj in list(minio.list_objects(bucket, recursive=True)): + if str(obj.object_name).find(".SCHEMA_VERSION") != -1: + continue + minio.remove_object(bucket, obj.object_name) + + +def drop_s3_metadata(node): + node.exec_in_container(['bash', '-c', 'rm -rf /var/lib/clickhouse/disks/s3/*'], user='root') + + +def drop_shadow_information(node): + node.exec_in_container(['bash', '-c', 'rm -rf /var/lib/clickhouse/shadow/*'], user='root') + + +def create_restore_file(node, revision=None, bucket=None, path=None, detached=None): + node.exec_in_container(['bash', '-c', 'mkdir -p /var/lib/clickhouse/disks/s3/'], user='root') + node.exec_in_container(['bash', '-c', 'touch /var/lib/clickhouse/disks/s3/restore'], user='root') + + add_restore_option = 'echo -en "{}={}\n" >> /var/lib/clickhouse/disks/s3/restore' + if revision: + node.exec_in_container(['bash', '-c', add_restore_option.format('revision', revision)], user='root') + if bucket: + node.exec_in_container(['bash', '-c', add_restore_option.format('source_bucket', bucket)], user='root') + if path: + node.exec_in_container(['bash', '-c', add_restore_option.format('source_path', path)], user='root') + if detached: + node.exec_in_container(['bash', '-c', add_restore_option.format('detached', 'true')], user='root') + + +def get_revision_counter(node, backup_number): + return int(node.exec_in_container( + ['bash', '-c', 'cat /var/lib/clickhouse/disks/s3/shadow/{}/revision.txt'.format(backup_number)], user='root')) + + +def get_table_uuid(node, db_atomic, table): + uuid = "" + if db_atomic: + uuid = node.query("SELECT uuid FROM system.tables WHERE database='s3' AND table='{}' FORMAT TabSeparated".format(table)).strip() + return uuid + + +@pytest.fixture(autouse=True) +def drop_table(cluster): + yield + + node_names = ["node1z", "node2z", "node1n", "node2n", "node_another_bucket"] + + for node_name in node_names: + node = cluster.instances[node_name] + node.query("DROP TABLE IF EXISTS s3.test SYNC") + node.query("DROP DATABASE IF EXISTS s3 SYNC") + + drop_s3_metadata(node) + drop_shadow_information(node) + + buckets = [cluster.minio_bucket, cluster.minio_bucket_2] + for bucket in buckets: + purge_s3(cluster, bucket) + + +@pytest.mark.parametrize( + "db_atomic", [False, True] +) +@pytest.mark.parametrize( + "zero_copy", [False, True] +) +def test_restore_another_bucket_path(cluster, db_atomic, zero_copy): + suffix = "z" if zero_copy else "n" + nodes = [cluster.instances[f"node1{suffix}"], cluster.instances[f"node2{suffix}"]] + + keys = 100 + data_columns = 10 + size = 1 + + columns = [] + for c in range(0, data_columns): + columns.append("data{c} String".format(c=c)) + schema = ", ".join(columns) + + create_table(nodes[0], "test", schema, db_atomic=db_atomic) + uuid = get_table_uuid(nodes[0], db_atomic, "test") + + + dropped_keys = 0 + + for key in range(0, keys): + node = nodes[key % 2] + node.query("INSERT INTO s3.test SELECT {key}, * FROM generateRandom('{schema}') LIMIT {size}".format(key=key, schema=schema, size=size)) + if not (key % 3): + dropped_keys += 1 + node.query("ALTER TABLE s3.test DROP PARTITION '{key}'".format(key=key)) + + for key in range(0, keys): + if not ((key+1) % 3): + dropped_keys += 1 + node.query("ALTER TABLE s3.test DROP PARTITION '{key}'".format(key=key)) + + nodes[0].query("SYSTEM SYNC REPLICA s3.test") + nodes[1].query("SYSTEM SYNC REPLICA s3.test") + + # To ensure parts have merged + nodes[0].query("OPTIMIZE TABLE s3.test") + nodes[1].query("OPTIMIZE TABLE s3.test") + + assert nodes[0].query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(size * (keys - dropped_keys)) + assert nodes[1].query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(size * (keys - dropped_keys)) + + node_another_bucket = cluster.instances["node_another_bucket"] + + create_restore_file(node_another_bucket, bucket="root") + node_another_bucket.query("SYSTEM RESTART DISK s3") + create_table(node_another_bucket, "test", schema, attach=True, db_atomic=db_atomic, uuid=uuid) + + assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(size * (keys - dropped_keys)) + From b483888a033e87d37e9591820e0aec66b0a4a050 Mon Sep 17 00:00:00 2001 From: zvonand Date: Wed, 23 Mar 2022 22:21:35 +0300 Subject: [PATCH 0008/1954] added tests --- src/Functions/nonNegativeDerivative.h | 4 +--- .../0_stateless/02232_non_negative_derivative.sql | 11 +++++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/02232_non_negative_derivative.sql diff --git a/src/Functions/nonNegativeDerivative.h b/src/Functions/nonNegativeDerivative.h index 71949c7c991..b7214027fad 100644 --- a/src/Functions/nonNegativeDerivative.h +++ b/src/Functions/nonNegativeDerivative.h @@ -88,18 +88,16 @@ private: { auto cur = metric[i]; auto multiply = interval_length * ts_scale_multiplier / (timestamp[i].value - prev_ts.value); - result[i] = (cur - prev_metric_value) * multiply; + result[i] = cur >= prev_metric_value ? (cur - prev_metric_value) * multiply : 0; prev_metric_value = cur; prev_ts = timestamp[i]; } } } - /// Result type is same as result of subtraction of argument types. template using DstFieldType = typename NumberTraits::ResultOfFloatingPointDivision::Type; - /// Call polymorphic lambda with tag argument of concrete field type of src_type. template void dispatchForSourceType(const IDataType & src_type, F && f) const { diff --git a/tests/queries/0_stateless/02232_non_negative_derivative.sql b/tests/queries/0_stateless/02232_non_negative_derivative.sql new file mode 100644 index 00000000000..1b4c97a1982 --- /dev/null +++ b/tests/queries/0_stateless/02232_non_negative_derivative.sql @@ -0,0 +1,11 @@ +SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 1 SECOND) AS nnd +FROM ( + SELECT * FROM VALUES ( + 'ts DateTime64(3, "UTC"), metric Int32', + (toDateTime64('1979-12-12 21:21:21.123', 3, 'UTC'), 1), + (toDateTime64('1979-12-12 21:21:21.124', 3, 'UTC'), 2), + (toDateTime64('1979-12-12 21:21:21.127', 3, 'UTC'), 3), + (toDateTime64('1979-12-12 21:21:21.129', 3, 'UTC'), 2), + (toDateTime('1979-12-12 21:21:22', 'UTC'), 13) + ) + ); \ No newline at end of file From e47bc1c300dc38d8d11c7ef97b5353687cfad872 Mon Sep 17 00:00:00 2001 From: zvonand Date: Wed, 23 Mar 2022 22:32:30 +0300 Subject: [PATCH 0009/1954] updated tests --- .../02232_non_negative_derivative.reference | 12 ++++++++++++ .../02232_non_negative_derivative.sql | 18 ++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/02232_non_negative_derivative.reference diff --git a/tests/queries/0_stateless/02232_non_negative_derivative.reference b/tests/queries/0_stateless/02232_non_negative_derivative.reference new file mode 100644 index 00000000000..be942129a06 --- /dev/null +++ b/tests/queries/0_stateless/02232_non_negative_derivative.reference @@ -0,0 +1,12 @@ +1979-12-13 00:21:21.123 1 0 +1979-12-13 00:21:21.124 2 1000 +1979-12-13 00:21:21.127 3 333.3333333333333 +1979-12-13 00:21:21.129 2 0 +1979-12-13 00:21:22.000 13 12.629161882893225 +1979-12-13 00:21:23.000 10 0 +1979-12-13 00:21:21.123 1.1 0 +1979-12-13 00:21:21.124 2.34 1499903999.9999998 +1979-12-13 00:21:21.127 3.7 548352000.0000001 +1979-12-13 00:21:21.129 2.1 0 +1979-12-13 00:21:22.000 1.3345 0 +1979-12-13 00:21:23.000 1.5 200188.8 diff --git a/tests/queries/0_stateless/02232_non_negative_derivative.sql b/tests/queries/0_stateless/02232_non_negative_derivative.sql index 1b4c97a1982..fd9f86be60a 100644 --- a/tests/queries/0_stateless/02232_non_negative_derivative.sql +++ b/tests/queries/0_stateless/02232_non_negative_derivative.sql @@ -6,6 +6,20 @@ FROM ( (toDateTime64('1979-12-12 21:21:21.124', 3, 'UTC'), 2), (toDateTime64('1979-12-12 21:21:21.127', 3, 'UTC'), 3), (toDateTime64('1979-12-12 21:21:21.129', 3, 'UTC'), 2), - (toDateTime('1979-12-12 21:21:22', 'UTC'), 13) + (toDateTime('1979-12-12 21:21:22', 'UTC'), 13), + (toDateTime('1979-12-12 21:21:23', 'UTC'), 10) ) - ); \ No newline at end of file + ); + +SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 2 WEEK) AS nnd +FROM ( + SELECT * FROM VALUES ( + 'ts DateTime64(3, "UTC"), metric Float64', + (toDateTime64('1979-12-12 21:21:21.123', 3, 'UTC'), 1.1), + (toDateTime64('1979-12-12 21:21:21.124', 3, 'UTC'), 2.34), + (toDateTime64('1979-12-12 21:21:21.127', 3, 'UTC'), 3.7), + (toDateTime64('1979-12-12 21:21:21.129', 3, 'UTC'), 2.1), + (toDateTime('1979-12-12 21:21:22', 'UTC'), 1.3345), + (toDateTime('1979-12-12 21:21:23', 'UTC'), 1.5) + ) + ); \ No newline at end of file From 7f2ffb277eb9ab2ed5a9b0d9d828951cefb0b725 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 24 Mar 2022 02:36:11 +0100 Subject: [PATCH 0010/1954] Fix test --- tests/queries/0_stateless/01710_projections.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01710_projections.sql b/tests/queries/0_stateless/01710_projections.sql index 54581b5ae11..5df16a93980 100644 --- a/tests/queries/0_stateless/01710_projections.sql +++ b/tests/queries/0_stateless/01710_projections.sql @@ -40,6 +40,7 @@ select toStartOfMinute(datetime) dt_m, domain, sum(retry_count) / sum(duration), select toStartOfHour(toStartOfMinute(datetime)) dt_h, uniqHLL12(x_id), uniqHLL12(y_id) from projection_test group by dt_h order by dt_h; -- found by fuzzer +SET enable_positional_arguments = 0; SELECT 2, -1 FROM projection_test PREWHERE domain_alias = 1. WHERE domain = NULL GROUP BY -9223372036854775808 ORDER BY countIf(first_time = 0) / count(-2147483649) DESC NULLS LAST, 1048576 DESC NULLS LAST; drop table if exists projection_test; From 98012c32c88f69d1f6f9a8891d41fa651fe8063d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 24 Mar 2022 02:39:28 +0100 Subject: [PATCH 0011/1954] Fix test --- tests/queries/0_stateless/00209_insert_select_extremes.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/00209_insert_select_extremes.sql b/tests/queries/0_stateless/00209_insert_select_extremes.sql index 8c6e547131c..98dfe8e2658 100644 --- a/tests/queries/0_stateless/00209_insert_select_extremes.sql +++ b/tests/queries/0_stateless/00209_insert_select_extremes.sql @@ -1,6 +1,8 @@ DROP TABLE IF EXISTS test_00209; CREATE TABLE test_00209 (x UInt8) ENGINE = Log; +SET enable_positional_arguments = 0; + INSERT INTO test_00209 SELECT 1 AS x; INSERT INTO test_00209 SELECT 1 AS x SETTINGS extremes = 1; INSERT INTO test_00209 SELECT 1 AS x GROUP BY 1 WITH TOTALS; From 15ca875ba79a69347a5fb69d08980287f4421b65 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 24 Mar 2022 02:43:51 +0100 Subject: [PATCH 0012/1954] Give up with one setting --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 9e6ab27f251..1d1a255054a 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -453,7 +453,7 @@ class IColumn; M(Bool, optimize_if_chain_to_multiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \ M(Bool, optimize_if_transform_strings_to_enum, false, "Replaces string-type arguments in If and Transform to enum. Disabled by default cause it could make inconsistent change in distributed query that would lead to its fail.", 0) \ M(Bool, optimize_monotonous_functions_in_order_by, true, "Replace monotonous function with its argument in ORDER BY", 0) \ - M(Bool, optimize_functions_to_subcolumns, true, "Transform functions to subcolumns, if possible, to reduce amount of read data. E.g. 'length(arr)' -> 'arr.size0', 'col IS NULL' -> 'col.null' ", 0) \ + M(Bool, optimize_functions_to_subcolumns, false, "Transform functions to subcolumns, if possible, to reduce amount of read data. E.g. 'length(arr)' -> 'arr.size0', 'col IS NULL' -> 'col.null' ", 0) \ M(Bool, optimize_using_constraints, false, "Use constraints for query optimization", 0) \ M(Bool, optimize_substitute_columns, false, "Use constraints for column substitution", 0) \ M(Bool, optimize_append_index, false, "Use constraints in order to append index condition (indexHint)", 0) \ From add44ed9142715db5f2c0017e4bb6bf218739de6 Mon Sep 17 00:00:00 2001 From: zvonand Date: Thu, 24 Mar 2022 14:21:42 +0300 Subject: [PATCH 0013/1954] updated test --- src/Functions/nonNegativeDerivative.h | 6 ++ .../02232_non_negative_derivative.reference | 62 +++++++++++++++---- .../02232_non_negative_derivative.sql | 43 ++++++------- 3 files changed, 75 insertions(+), 36 deletions(-) diff --git a/src/Functions/nonNegativeDerivative.h b/src/Functions/nonNegativeDerivative.h index b7214027fad..92dd483cdbf 100644 --- a/src/Functions/nonNegativeDerivative.h +++ b/src/Functions/nonNegativeDerivative.h @@ -45,6 +45,12 @@ private: return interval_length * 60; case IntervalKind::Second: return interval_length; + case IntervalKind::Millisecond: + return interval_length * 0.001; + case IntervalKind::Microsecond: + return interval_length * 0.000001; + case IntervalKind::Nanosecond: + return interval_length * 0.000000001; default: throw Exception(fmt::format("Interval kind {}: interval length is variadic, only precise intervals accepted", IntervalKind(interval_kind).toKeyword()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); diff --git a/tests/queries/0_stateless/02232_non_negative_derivative.reference b/tests/queries/0_stateless/02232_non_negative_derivative.reference index be942129a06..575b50f89ab 100644 --- a/tests/queries/0_stateless/02232_non_negative_derivative.reference +++ b/tests/queries/0_stateless/02232_non_negative_derivative.reference @@ -1,12 +1,50 @@ -1979-12-13 00:21:21.123 1 0 -1979-12-13 00:21:21.124 2 1000 -1979-12-13 00:21:21.127 3 333.3333333333333 -1979-12-13 00:21:21.129 2 0 -1979-12-13 00:21:22.000 13 12.629161882893225 -1979-12-13 00:21:23.000 10 0 -1979-12-13 00:21:21.123 1.1 0 -1979-12-13 00:21:21.124 2.34 1499903999.9999998 -1979-12-13 00:21:21.127 3.7 548352000.0000001 -1979-12-13 00:21:21.129 2.1 0 -1979-12-13 00:21:22.000 1.3345 0 -1979-12-13 00:21:23.000 1.5 200188.8 +- shall work for precise intervals +1979-12-12 21:21:21.123 1.1 0 +1979-12-12 21:21:21.124 2.34 0.00000372 +1979-12-12 21:21:21.127 3.7 0.0000013600000000000005 +1979-12-12 21:21:21.129 2.1 0 +1979-12-12 21:21:22.000 1.3345 0 +1979-12-12 21:21:23.000 1.54 6.165000000000001e-10 +1979-12-12 21:21:21.123 1.1 0 +1979-12-12 21:21:21.124 2.34 0.004959999999999999 +1979-12-12 21:21:21.127 3.7 0.0018133333333333337 +1979-12-12 21:21:21.129 2.1 0 +1979-12-12 21:21:22.000 1.3345 0 +1979-12-12 21:21:23.000 1.54 8.22e-7 +1979-12-12 21:21:21.123 1.1 0 +1979-12-12 21:21:21.124 2.34 6.199999999999999 +1979-12-12 21:21:21.127 3.7 2.2666666666666675 +1979-12-12 21:21:21.129 2.1 0 +1979-12-12 21:21:22.000 1.3345 0 +1979-12-12 21:21:23.000 1.54 0.0010275000000000002 +1979-12-12 21:21:21.123 1.1 0 +1979-12-12 21:21:21.124 2.34 7439.999999999998 +1979-12-12 21:21:21.127 3.7 2720.0000000000005 +1979-12-12 21:21:21.129 2.1 0 +1979-12-12 21:21:22.000 1.3345 0 +1979-12-12 21:21:23.000 1.54 1.233 +1979-12-12 21:21:21.123 1.1 0 +1979-12-12 21:21:21.124 2.34 520799.9999999999 +1979-12-12 21:21:21.127 3.7 190400.00000000006 +1979-12-12 21:21:21.129 2.1 0 +1979-12-12 21:21:22.000 1.3345 0 +1979-12-12 21:21:23.000 1.54 86.31 +1979-12-12 21:21:21.123 1.1 0 +1979-12-12 21:21:21.124 2.34 35711999.99999999 +1979-12-12 21:21:21.127 3.7 13056000.000000004 +1979-12-12 21:21:21.129 2.1 0 +1979-12-12 21:21:22.000 1.3345 0 +1979-12-12 21:21:23.000 1.54 5918.400000000001 +1979-12-12 21:21:21.123 1.1 0 +1979-12-12 21:21:21.124 2.34 964223999.9999998 +1979-12-12 21:21:21.127 3.7 352512000.00000006 +1979-12-12 21:21:21.129 2.1 0 +1979-12-12 21:21:22.000 1.3345 0 +1979-12-12 21:21:23.000 1.54 159796.80000000002 +1979-12-12 21:21:21.123 1.1 0 +1979-12-12 21:21:21.124 2.34 7499519999.999999 +1979-12-12 21:21:21.127 3.7 2741760000.0000005 +1979-12-12 21:21:21.129 2.1 0 +1979-12-12 21:21:22.000 1.3345 0 +1979-12-12 21:21:23.000 1.54 1242864 +- shall not work for month, quarter, year diff --git a/tests/queries/0_stateless/02232_non_negative_derivative.sql b/tests/queries/0_stateless/02232_non_negative_derivative.sql index fd9f86be60a..fe25985315b 100644 --- a/tests/queries/0_stateless/02232_non_negative_derivative.sql +++ b/tests/queries/0_stateless/02232_non_negative_derivative.sql @@ -1,25 +1,20 @@ -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 1 SECOND) AS nnd -FROM ( - SELECT * FROM VALUES ( - 'ts DateTime64(3, "UTC"), metric Int32', - (toDateTime64('1979-12-12 21:21:21.123', 3, 'UTC'), 1), - (toDateTime64('1979-12-12 21:21:21.124', 3, 'UTC'), 2), - (toDateTime64('1979-12-12 21:21:21.127', 3, 'UTC'), 3), - (toDateTime64('1979-12-12 21:21:21.129', 3, 'UTC'), 2), - (toDateTime('1979-12-12 21:21:22', 'UTC'), 13), - (toDateTime('1979-12-12 21:21:23', 'UTC'), 10) - ) - ); +DROP TABLE IF EXISTS nnd; +CREATE TABLE nnd (id Int8, ts DateTime64(3, 'UTC'), metric Float64) ENGINE=MergeTree() ORDER BY id; +INSERT INTO nnd VALUES (1, toDateTime64('1979-12-12 21:21:21.123', 3, 'UTC'), 1.1), (2, toDateTime64('1979-12-12 21:21:21.124', 3, 'UTC'), 2.34), (3, toDateTime64('1979-12-12 21:21:21.127', 3, 'UTC'), 3.7), (4, toDateTime64('1979-12-12 21:21:21.129', 3, 'UTC'), 2.1), (5, toDateTime('1979-12-12 21:21:22', 'UTC'), 1.3345), (6, toDateTime('1979-12-12 21:21:23', 'UTC'), 1.54); -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 2 WEEK) AS nnd -FROM ( - SELECT * FROM VALUES ( - 'ts DateTime64(3, "UTC"), metric Float64', - (toDateTime64('1979-12-12 21:21:21.123', 3, 'UTC'), 1.1), - (toDateTime64('1979-12-12 21:21:21.124', 3, 'UTC'), 2.34), - (toDateTime64('1979-12-12 21:21:21.127', 3, 'UTC'), 3.7), - (toDateTime64('1979-12-12 21:21:21.129', 3, 'UTC'), 2.1), - (toDateTime('1979-12-12 21:21:22', 'UTC'), 1.3345), - (toDateTime('1979-12-12 21:21:23', 'UTC'), 1.5) - ) - ); \ No newline at end of file +SELECT '- shall work for precise intervals'; +SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 3 NANOSECOND) FROM nnd; +SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 4 MICROSECOND) FROM nnd; +SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 5 MILLISECOND) FROM nnd; +SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 6 SECOND) FROM nnd; +SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 7 MINUTE) FROM nnd; +SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 8 HOUR) FROM nnd; +SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 9 DAY) FROM nnd; +SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 10 WEEK) FROM nnd; + +SELECT '- shall not work for month, quarter, year'; +SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 11 MONTH) FROM nnd; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 12 QUARTER) FROM nnd; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 13 YEAR) FROM nnd; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +DROP TABLE IF EXISTS nnd; From 43c4f3cf84448657f40c62600ef6c92e998fa843 Mon Sep 17 00:00:00 2001 From: zvonand Date: Thu, 24 Mar 2022 15:56:41 +0300 Subject: [PATCH 0014/1954] updated docs --- .../functions/other-functions.md | 48 +++++++++++++++++++ .../functions/other-functions.md | 46 ++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index bce3f9144b1..dd4c29cbcd2 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -914,6 +914,54 @@ WHERE diff != 1 Same as for [runningDifference](./other-functions.md#other_functions-runningdifference), the difference is the value of the first row, returned the value of the first row, and each subsequent row returns the difference from the previous row. +## nonNegativeDerivative(metric, timestamp[, INTERVAL]) {#other_functions-nonnegativederivative} + +Calculates the growth ratio of given metric values by given timestamps. +Returns 0 for the first row and derivative for all the consecutive rows. +If at some point derivative is less than zero, this value is ignored and the result is 0. + +!!! warning "Warning" +It can reach the previous row only inside the currently processed data block. +The result of the function depends on the affected data blocks and the order of data in the block. + +!!! warning "Warning" +The rows order used during the calculation of `nonNegativeDerivative` can differ from the order of rows returned to the user. +To prevent that you can make a subquery with [ORDER BY](../../sql-reference/statements/select/order-by.md) and call the function from outside the subquery. + +Example: + +``` sql +SELECT + ts, + metric, + nonNegativeDerivative(metric, ts, toIntervalNanosecond(3)) AS nnd +FROM +( + SELECT * + FROM + values('id Int32, ts DateTime(3), metric Float64', + (1, toDateTime64('1979-12-12 21:21:21.123', 3, 'UTC'), 1.1), + (2, toDateTime64('1979-12-12 21:21:21.124', 3, 'UTC'), 2.34), + (3, toDateTime64('1979-12-12 21:21:21.127', 3, 'UTC'), 3.7), + (4, toDateTime64('1979-12-12 21:21:21.129', 3, 'UTC'), 2.1)) +) + +``` + +``` text +┌──────────────────────ts─┬─metric─┬──────────────────────nnd─┐ +│ 1979-12-13 00:21:21.123 │ 1.1 │ 0 │ +│ 1979-12-13 00:21:21.124 │ 2.34 │ 0.00000372 │ +│ 1979-12-13 00:21:21.127 │ 3.7 │ 0.0000013600000000000005 │ +│ 1979-12-13 00:21:21.129 │ 2.1 │ 0 │ +└─────────────────────────┴────────┴──────────────────────────┘ + +``` + +!!! note "Note" +Like with [runningDifference](#runningdifferencex-other_functions-runningdifference), block size affects the result. With each new block, the function state is reset. + + ## runningConcurrency {#runningconcurrency} Calculates the number of concurrent events. diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 62f25c221c5..62776400a84 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -859,6 +859,52 @@ WHERE diff != 1 То же, что и [runningDifference](./other-functions.md#other_functions-runningdifference), но в первой строке возвращается значение первой строки, а не ноль. +## nonNegativeDerivative(metric, timestamp[, INTERVAL]) {#other_functions-nonnegativederivative} + +В каждый промежуток времени между моментами в колонке `timestamp` считает прирост функции относительно указанного интервала. +Для первого ряда это значение 0. Если какое-либо из полученных значений отрицательно, то результатом будет 0. + +!!! warning "Warning" +Имейте в виду, что значение производной может быть посчитано только внутри одного блока данных. +Результат вычисления зависит от порядка данных внутри блока. + +!!! warning "Warning" +Порядок рядов, используемый при работе функции, может изменяться от того, что возвращается пользователю. +Дабы избежать этого, используйте [ORDER BY](../../sql-reference/statements/select/order-by.md). + +Пример использования: + +``` sql +SELECT + ts, + metric, + nonNegativeDerivative(metric, ts, toIntervalNanosecond(3)) AS nnd +FROM +( + SELECT * + FROM + values('id Int32, ts DateTime(3), metric Float64', + (1, toDateTime64('1979-12-12 21:21:21.123', 3, 'UTC'), 1.1), + (2, toDateTime64('1979-12-12 21:21:21.124', 3, 'UTC'), 2.34), + (3, toDateTime64('1979-12-12 21:21:21.127', 3, 'UTC'), 3.7), + (4, toDateTime64('1979-12-12 21:21:21.129', 3, 'UTC'), 2.1)) +) + +``` + +``` text +┌──────────────────────ts─┬─metric─┬──────────────────────nnd─┐ +│ 1979-12-13 00:21:21.123 │ 1.1 │ 0 │ +│ 1979-12-13 00:21:21.124 │ 2.34 │ 0.00000372 │ +│ 1979-12-13 00:21:21.127 │ 3.7 │ 0.0000013600000000000005 │ +│ 1979-12-13 00:21:21.129 │ 2.1 │ 0 │ +└─────────────────────────┴────────┴──────────────────────────┘ + +``` + +!!! note "Note" +Как и в случае с [runningDifference](#runningdifferencex-other_functions-runningdifference), размер блока влияет на результат. С каждым новым блоком состояние функции обнуляется. + ## runningConcurrency {#runningconcurrency} Подсчитывает количество одновременно идущих событий. From 38df1e945b243abd3d6a7c3b22f9709f202f8fea Mon Sep 17 00:00:00 2001 From: zvonand Date: Thu, 24 Mar 2022 16:36:11 +0300 Subject: [PATCH 0015/1954] fix style --- src/Functions/nonNegativeDerivative.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/nonNegativeDerivative.h b/src/Functions/nonNegativeDerivative.h index 92dd483cdbf..88166345f54 100644 --- a/src/Functions/nonNegativeDerivative.h +++ b/src/Functions/nonNegativeDerivative.h @@ -21,6 +21,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ARGUMENT_OUT_OF_BOUND; } /** Calculate derivative of given value column by specified timestamp. */ From 3c2ce079cbc430acdb730d93ba90d65ab8572e28 Mon Sep 17 00:00:00 2001 From: zvonand Date: Thu, 24 Mar 2022 17:35:53 +0300 Subject: [PATCH 0016/1954] scope fix --- src/Functions/nonNegativeDerivative.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Functions/nonNegativeDerivative.h b/src/Functions/nonNegativeDerivative.h index 88166345f54..13f1735b8ff 100644 --- a/src/Functions/nonNegativeDerivative.h +++ b/src/Functions/nonNegativeDerivative.h @@ -54,7 +54,7 @@ private: return interval_length * 0.000000001; default: throw Exception(fmt::format("Interval kind {}: interval length is variadic, only precise intervals accepted", - IntervalKind(interval_kind).toKeyword()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + IntervalKind(interval_kind).toKeyword()), DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } } @@ -137,7 +137,7 @@ private: else if (which.isDateTime()) f(DataTypeDateTime::FieldType()); else - throw Exception("First argument for function " + getName() + " must have numeric type.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("First argument for function " + getName() + " must have numeric type.", DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } public: @@ -188,17 +188,17 @@ public: const auto * interval_type = checkAndGetDataType(interval_column.type.get()); if (!interval_type) throw Exception("Illegal value" + interval_column.name + "for function nonNegativeDerivative, INTERVAL expected", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); const auto * interval_column_const_int64 = checkAndGetColumnConst(interval_column.column.get()); if (!interval_column_const_int64) throw Exception("Illegal value " + interval_column.name + "for function nonNegativeDerivative, INTERVAL expected", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); Int64 num_units = interval_column_const_int64->getValue(); if (num_units <= 0) throw Exception("Value for column " + interval_column.name + "for function nonNegativeDerivative must be positive", - ErrorCodes::ARGUMENT_OUT_OF_BOUND); + DB::ErrorCodes::ARGUMENT_OUT_OF_BOUND); return {interval_type->getKind(), num_units}; } From c01aa3e4fbe5dd8c8dcbb3d17456bf260059a293 Mon Sep 17 00:00:00 2001 From: zvonand Date: Thu, 24 Mar 2022 19:33:24 +0300 Subject: [PATCH 0017/1954] retrigger checks From 5a40d868ea5a6ba1280c6c69f3735430ab0539e1 Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Thu, 24 Mar 2022 22:45:54 +0300 Subject: [PATCH 0018/1954] add compressor --- utils/CMakeLists.txt | 1 + utils/self-extr-exec/CMakeLists.txt | 5 + utils/self-extr-exec/compressor.cpp | 141 ++++++++++++++++++++++++++ utils/self-extr-exec/decompressor.cpp | 7 ++ 4 files changed, 154 insertions(+) create mode 100644 utils/self-extr-exec/CMakeLists.txt create mode 100644 utils/self-extr-exec/compressor.cpp create mode 100644 utils/self-extr-exec/decompressor.cpp diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 51300472ed1..11334988957 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -32,6 +32,7 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) add_subdirectory (check-mysql-binlog) add_subdirectory (keeper-bench) add_subdirectory (graphite-rollup) + add_subdirectory (self-extr-exec) if (TARGET ch_contrib::nuraft) add_subdirectory (keeper-data-dumper) diff --git a/utils/self-extr-exec/CMakeLists.txt b/utils/self-extr-exec/CMakeLists.txt new file mode 100644 index 00000000000..0054996dd82 --- /dev/null +++ b/utils/self-extr-exec/CMakeLists.txt @@ -0,0 +1,5 @@ +add_executable (compressor compressor.cpp) +target_link_libraries(compressor PUBLIC ch_contrib::zstd) + +add_executable (decompressor decompressor.cpp) +target_link_libraries(compressor PUBLIC ch_contrib::zstd) diff --git a/utils/self-extr-exec/compressor.cpp b/utils/self-extr-exec/compressor.cpp new file mode 100644 index 00000000000..bf0b2c6c297 --- /dev/null +++ b/utils/self-extr-exec/compressor.cpp @@ -0,0 +1,141 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +/// Main compression part +int doCompress(char * input, char * output, off_t & in_offset, off_t & out_offset, + off_t input_size, off_t output_size, ZSTD_CCtx * cctx) +{ + size_t compressed_size = ZSTD_compress2(cctx, output + out_offset, output_size, input + in_offset, input_size); + if (ZSTD_isError(compressed_size)) + { + std::cout << "Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(compressed_size)) << std::endl; + return 1; + } + in_offset += input_size; + out_offset += output_size; + return 0; +} + +/// compress data from opened file into output file +int compress(int in_fd, int out_fd, int level=3) +{ + /// read data about input file + struct stat info_in; + fstat(in_fd, &info_in); + if (info_in.st_size == 0) { + std::cout << "Empty input file" << std::endl; + return 1; + } + + /// Read data about output file. + /// Compressed data will be added to the end of file + /// It will allow to create self extracting executable from file + struct stat info_out; + fstat(out_fd, &info_out); + + /// As experiments showed, size of compressed file is 4 times less than clickhouse executable + /// Get a little bit more memory to prevent errors with size. + /// For compression this difference will not be huge + ftruncate(out_fd, info_out.st_size + info_in.st_size / 3); + off_t in_offset = 0, out_offset = info_out.st_size; + + /// mmap files + char * input = static_cast(mmap(nullptr, info_in.st_size, PROT_READ | PROT_EXEC, MAP_SHARED , in_fd, 0)); + char * output = static_cast(mmap(nullptr, info_out.st_size + info_in.st_size / 3, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_SHARED, out_fd, 0)); + if (input == reinterpret_cast(-1) || output == reinterpret_cast(-1)) + { + std::cout << (input == reinterpret_cast(-1)) << " " << (output == reinterpret_cast(-1)) << std::endl; + perror(nullptr); + return 1; + } + + /// Create context + ZSTD_CCtx * cctx = ZSTD_createCCtx(); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); + // ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, 9); + + /// Remember size of file. It will help to avoid using additional memory + /// during decompression + // char * file_size = reinterpret_cast(&info_in.st_size); + // for (size_t i = 0; i < sizeof(info_in.st_size)/sizeof(char); ++i) + // output[out_offset++] = *file_size; + + + /// limits for size of block to prevent high memory usage or bad compression + // off_t max_block_size = 100000000ull; + // off_t min_block_size = 10000000ull; + off_t size = 0; + + /// Compress data + while (in_offset < info_in.st_size) + { + /// take blocks of maximum size + /// optimize last block (it can be bigger, if it is not too huge) + // if (info_in.st_size - in_offset < max_block_size || info_in.st_size - in_offset < max_block_size + min_block_size) + // size = info_in.st_size - in_offset; + // else + // size = max_block_size; + size = info_in.st_size - in_offset; + + /// Compress data or exit if error happens + if (0 != doCompress(input, output, in_offset, out_offset, size, ZSTD_compressBound(size), cctx)) + { + ftruncate(out_fd, info_out.st_size); + munmap(input, info_in.st_size); + munmap(output, info_out.st_size + info_in.st_size / 3); + return 1; + } + std::cout << in_offset << " " << out_offset << std::endl; + } + + /// Shrink file size and unmap + ftruncate(out_fd, out_offset); + munmap(input, info_in.st_size); + munmap(output, info_out.st_size + info_in.st_size / 3); + return 0; +} + +int main(int argc, char* argv[]) +{ + if (argc < 3) + { + std::cout << "Not enough arguments.\ncompressor [file that should be compressed] [file name for compressed file] [OPTIONAL level of compression]" << std::endl; + return 0; + } + + int input_fd = open(argv[1], O_RDWR); + if (input_fd == -1) + { + perror(nullptr); + return 0; + } + + int output_fd = open(argv[2], O_RDWR | O_CREAT, 0777); + if (input_fd == -1) + { + perror(nullptr); + return 0; + } + + int result; + if (argc == 4) + result = compress(input_fd, output_fd, strtol(argv[3], nullptr, 10)); + else + result = compress(input_fd, output_fd); + + if (result == 0) + std::cout << "Successfully compressed" << std::endl; + else + std::cout << "An error has occurred" << std::endl; + + close(input_fd); + close(output_fd); + return 0; +} diff --git a/utils/self-extr-exec/decompressor.cpp b/utils/self-extr-exec/decompressor.cpp new file mode 100644 index 00000000000..5f0a467aef7 --- /dev/null +++ b/utils/self-extr-exec/decompressor.cpp @@ -0,0 +1,7 @@ +#include + +int main() +{ + std::cout << "Test" << std::endl; + return 0; +} From 28e5d8bef8b6bed858e4a77a8438b1e6fe54d958 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 24 Mar 2022 21:31:49 +0100 Subject: [PATCH 0019/1954] Disable trash optimizations --- src/Core/Settings.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 1d1a255054a..593bcab43d6 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -484,8 +484,8 @@ class IColumn; M(Bool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \ M(Bool, enable_global_with_statement, true, "Propagate WITH statements to UNION queries and all subqueries", 0) \ M(Bool, aggregate_functions_null_for_empty, false, "Rewrite all aggregate functions in a query, adding -OrNull suffix to them", 0) \ - M(Bool, optimize_syntax_fuse_functions, true, "Allow apply syntax optimisation: fuse aggregate functions", 0) \ - M(Bool, optimize_fuse_sum_count_avg, true, "Fuse functions `sum, avg, count` with identical arguments into one `sumCount` (`optimize_syntax_fuse_functions should be enabled)", 0) \ + M(Bool, optimize_syntax_fuse_functions, false, "Not ready for production, do not use. Allow apply syntax optimisation: fuse aggregate functions", 0) \ + M(Bool, optimize_fuse_sum_count_avg, false, "Not ready for production, do not use. Fuse functions `sum, avg, count` with identical arguments into one `sumCount` (`optimize_syntax_fuse_functions should be enabled)", 0) \ M(Bool, flatten_nested, true, "If true, columns of type Nested will be flatten to separate array columns instead of one array of tuples", 0) \ M(Bool, asterisk_include_materialized_columns, false, "Include MATERIALIZED columns for wildcard query", 0) \ M(Bool, asterisk_include_alias_columns, false, "Include ALIAS columns for wildcard query", 0) \ From 642082826d6d45ff3468d314b73a9462204ea3e2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 24 Mar 2022 21:31:59 +0100 Subject: [PATCH 0020/1954] Update test --- .../00019_shard_quantiles_totals_distributed.sql | 3 ++- tests/queries/0_stateless/01162_strange_mutations.sh | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/00019_shard_quantiles_totals_distributed.sql b/tests/queries/0_stateless/00019_shard_quantiles_totals_distributed.sql index 84956f0d97e..e712b028a82 100644 --- a/tests/queries/0_stateless/00019_shard_quantiles_totals_distributed.sql +++ b/tests/queries/0_stateless/00019_shard_quantiles_totals_distributed.sql @@ -1,3 +1,4 @@ -- Tags: distributed -SELECT quantilesTiming(0.1, 0.5, 0.9)(dummy) FROM remote('127.0.0.{2,3}', system, one) GROUP BY 1 WITH TOTALS +SET enable_positional_arguments = 0; +SELECT quantilesTiming(0.1, 0.5, 0.9)(dummy) FROM remote('127.0.0.{2,3}', system, one) GROUP BY 1 WITH TOTALS; diff --git a/tests/queries/0_stateless/01162_strange_mutations.sh b/tests/queries/0_stateless/01162_strange_mutations.sh index c759d113f84..ce75b584189 100755 --- a/tests/queries/0_stateless/01162_strange_mutations.sh +++ b/tests/queries/0_stateless/01162_strange_mutations.sh @@ -28,15 +28,15 @@ do $CLICKHOUSE_CLIENT -q "CREATE TABLE test ENGINE=$engine AS SELECT number + 100 AS n, 0 AS test FROM numbers(50)" 2>&1| grep -Ev "Removing leftovers from table|removed by another replica" $CLICKHOUSE_CLIENT -q "select count(), sum(n), sum(test) from test" if [[ $engine == *"ReplicatedMergeTree"* ]]; then - $CLICKHOUSE_CLIENT -q "ALTER TABLE test + $CLICKHOUSE_CLIENT --enable_positional_arguments 0 -q "ALTER TABLE test UPDATE test = (SELECT groupArray(id) FROM t1 GROUP BY 1)[n - 99] WHERE 1" 2>&1| grep -Fa "DB::Exception: " | grep -Fv "statement with subquery may be nondeterministic" - $CLICKHOUSE_CLIENT --allow_nondeterministic_mutations=1 --mutations_sync=1 -q "ALTER TABLE test + $CLICKHOUSE_CLIENT --enable_positional_arguments 0 --allow_nondeterministic_mutations=1 --mutations_sync=1 -q "ALTER TABLE test UPDATE test = (SELECT groupArray(id) FROM t1 GROUP BY 1)[n - 99] WHERE 1" elif [[ $engine == *"Join"* ]]; then - $CLICKHOUSE_CLIENT -q "ALTER TABLE test + $CLICKHOUSE_CLIENT --enable_positional_arguments 0 -q "ALTER TABLE test UPDATE test = (SELECT groupArray(id) FROM t1 GROUP BY 1)[n - 99] WHERE 1" 2>&1| grep -Fa "DB::Exception: " | grep -Fv "Table engine Join supports only DELETE mutations" else - $CLICKHOUSE_CLIENT --mutations_sync=1 -q "ALTER TABLE test + $CLICKHOUSE_CLIENT --enable_positional_arguments 0 --mutations_sync=1 -q "ALTER TABLE test UPDATE test = (SELECT groupArray(id) FROM t1 GROUP BY 1)[n - 99] WHERE 1" fi $CLICKHOUSE_CLIENT -q "select count(), sum(n), sum(test) from test" From 58c5dcfd125d6e758c70f298b69d2924e9429d2a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 24 Mar 2022 21:36:41 +0100 Subject: [PATCH 0021/1954] Disable one more trash setting --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 593bcab43d6..916f156c034 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -129,7 +129,7 @@ class IColumn; M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \ M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \ M(UInt64, aggregation_memory_efficient_merge_threads, 0, "Number of threads to use for merge intermediate aggregation results in memory efficient mode. When bigger, then more memory is consumed. 0 means - same as 'max_threads'.", 0) \ - M(Bool, enable_positional_arguments, true, "Enable positional arguments in ORDER BY, GROUP BY and LIMIT BY", 0) \ + M(Bool, enable_positional_arguments, false, "Not ready for production, do not use. Enable positional arguments in ORDER BY, GROUP BY and LIMIT BY", 0) \ \ M(UInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled.", 0) \ M(UInt64, parallel_replicas_count, 0, "", 0) \ From 7f0c6fb103727662e63f5a90a626f1092d090883 Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Tue, 29 Mar 2022 19:30:09 +0000 Subject: [PATCH 0022/1954] improve self-extr-executable --- utils/self-extr-exec/CMakeLists.txt | 2 +- utils/self-extr-exec/compressor.cpp | 55 +++++++----- utils/self-extr-exec/decompressor.cpp | 106 ++++++++++++++++++++++- utils/self-extr-exec/make_compression.sh | 9 ++ 4 files changed, 145 insertions(+), 27 deletions(-) create mode 100755 utils/self-extr-exec/make_compression.sh diff --git a/utils/self-extr-exec/CMakeLists.txt b/utils/self-extr-exec/CMakeLists.txt index 0054996dd82..b804fc27e72 100644 --- a/utils/self-extr-exec/CMakeLists.txt +++ b/utils/self-extr-exec/CMakeLists.txt @@ -2,4 +2,4 @@ add_executable (compressor compressor.cpp) target_link_libraries(compressor PUBLIC ch_contrib::zstd) add_executable (decompressor decompressor.cpp) -target_link_libraries(compressor PUBLIC ch_contrib::zstd) +target_link_libraries(decompressor PUBLIC ch_contrib::zstd) diff --git a/utils/self-extr-exec/compressor.cpp b/utils/self-extr-exec/compressor.cpp index bf0b2c6c297..951117ec8be 100644 --- a/utils/self-extr-exec/compressor.cpp +++ b/utils/self-extr-exec/compressor.cpp @@ -18,12 +18,12 @@ int doCompress(char * input, char * output, off_t & in_offset, off_t & out_offse return 1; } in_offset += input_size; - out_offset += output_size; + out_offset += compressed_size; return 0; } /// compress data from opened file into output file -int compress(int in_fd, int out_fd, int level=3) +int compress(int in_fd, int out_fd, int level=5) { /// read data about input file struct stat info_in; @@ -32,25 +32,33 @@ int compress(int in_fd, int out_fd, int level=3) std::cout << "Empty input file" << std::endl; return 1; } + std::cout << "In current size is " << info_in.st_size << std::endl; /// Read data about output file. /// Compressed data will be added to the end of file /// It will allow to create self extracting executable from file struct stat info_out; - fstat(out_fd, &info_out); + fstat(out_fd, &info_out); + std::cout << "Out current size is " << info_out.st_size << std::endl; + + /// NOTE: next parametrs depend on binary size + // 6402520 is size of stripped decompressor + size_t start = 6405000ull; + + // 22550744 size of decompressor + // size_t start = 22550780ull; /// As experiments showed, size of compressed file is 4 times less than clickhouse executable /// Get a little bit more memory to prevent errors with size. /// For compression this difference will not be huge - ftruncate(out_fd, info_out.st_size + info_in.st_size / 3); - off_t in_offset = 0, out_offset = info_out.st_size; + ftruncate(out_fd, start + info_in.st_size / 3); + off_t in_offset = 0, out_offset = start; /// mmap files - char * input = static_cast(mmap(nullptr, info_in.st_size, PROT_READ | PROT_EXEC, MAP_SHARED , in_fd, 0)); - char * output = static_cast(mmap(nullptr, info_out.st_size + info_in.st_size / 3, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_SHARED, out_fd, 0)); + char * input = static_cast(mmap(nullptr, info_in.st_size, PROT_READ, MAP_PRIVATE , in_fd, 0)); + char * output = static_cast(mmap(nullptr, start + info_in.st_size / 3, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_SHARED, out_fd, 0)); if (input == reinterpret_cast(-1) || output == reinterpret_cast(-1)) { - std::cout << (input == reinterpret_cast(-1)) << " " << (output == reinterpret_cast(-1)) << std::endl; perror(nullptr); return 1; } @@ -59,37 +67,36 @@ int compress(int in_fd, int out_fd, int level=3) ZSTD_CCtx * cctx = ZSTD_createCCtx(); ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level); ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); - // ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, 9); /// Remember size of file. It will help to avoid using additional memory /// during decompression - // char * file_size = reinterpret_cast(&info_in.st_size); - // for (size_t i = 0; i < sizeof(info_in.st_size)/sizeof(char); ++i) - // output[out_offset++] = *file_size; + char * file_size = reinterpret_cast(&info_in.st_size); + for (size_t i = 0; i < sizeof(info_in.st_size)/sizeof(char); ++i) + output[out_offset++] = *(file_size + i); /// limits for size of block to prevent high memory usage or bad compression - // off_t max_block_size = 100000000ull; - // off_t min_block_size = 10000000ull; + off_t max_block_size = 1ull<<27; + off_t min_block_size = 1ull<<23; off_t size = 0; + std::cout << in_offset << " " << out_offset << std::endl; /// Compress data while (in_offset < info_in.st_size) { /// take blocks of maximum size /// optimize last block (it can be bigger, if it is not too huge) - // if (info_in.st_size - in_offset < max_block_size || info_in.st_size - in_offset < max_block_size + min_block_size) - // size = info_in.st_size - in_offset; - // else - // size = max_block_size; - size = info_in.st_size - in_offset; + if (info_in.st_size - in_offset < max_block_size || info_in.st_size - in_offset < max_block_size + min_block_size) + size = info_in.st_size - in_offset; + else + size = max_block_size; /// Compress data or exit if error happens if (0 != doCompress(input, output, in_offset, out_offset, size, ZSTD_compressBound(size), cctx)) { ftruncate(out_fd, info_out.st_size); munmap(input, info_in.st_size); - munmap(output, info_out.st_size + info_in.st_size / 3); + munmap(output, start + info_in.st_size / 3); return 1; } std::cout << in_offset << " " << out_offset << std::endl; @@ -98,7 +105,7 @@ int compress(int in_fd, int out_fd, int level=3) /// Shrink file size and unmap ftruncate(out_fd, out_offset); munmap(input, info_in.st_size); - munmap(output, info_out.st_size + info_in.st_size / 3); + munmap(output, start + info_in.st_size / 3); return 0; } @@ -110,15 +117,15 @@ int main(int argc, char* argv[]) return 0; } - int input_fd = open(argv[1], O_RDWR); + int input_fd = open(argv[1], O_RDONLY); if (input_fd == -1) { perror(nullptr); return 0; } - int output_fd = open(argv[2], O_RDWR | O_CREAT, 0777); - if (input_fd == -1) + int output_fd = open(argv[2], O_RDWR | O_CREAT, 0775); + if (output_fd == -1) { perror(nullptr); return 0; diff --git a/utils/self-extr-exec/decompressor.cpp b/utils/self-extr-exec/decompressor.cpp index 5f0a467aef7..6d6b44065ee 100644 --- a/utils/self-extr-exec/decompressor.cpp +++ b/utils/self-extr-exec/decompressor.cpp @@ -1,7 +1,109 @@ -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/// decompress part +int doDecompress(char * input, char * output, off_t & in_offset, off_t & out_offset, + off_t input_size, off_t output_size, ZSTD_DCtx* dctx) +{ + size_t decompressed_size = ZSTD_decompressDCtx(dctx, output + out_offset, output_size, input + in_offset, input_size); + if (ZSTD_isError(decompressed_size)) + { + return 1; + } + in_offset += input_size; + out_offset += decompressed_size; + return 0; +} + +/// decompress data from in_fd into out_fd +int decompress(int in_fd, int out_fd) +{ + /// Read data about output file. + /// Compressed data will replace data in file + struct stat info_in; + fstat(in_fd, &info_in); + + /// NOTE: next parametrs depend on binary size + // 22550780ull for full, 6405000ull for stripped; + off_t in_offset = 6405000ull /*size of decompressor*/, out_offset = 0; + + /// mmap files + char * input = static_cast(mmap(nullptr, info_in.st_size, PROT_READ, MAP_SHARED , in_fd, 0)); + if (input == reinterpret_cast(-1)) + { + perror(nullptr); + return 1; + } + + /// Create context + ZSTD_DCtx * dctx = ZSTD_createDCtx(); + + /// Read size of file. It will help to avoid using additional memory + /// during decompression. + size_t * file_size = reinterpret_cast(input + in_offset); + in_offset += sizeof(size_t); + + /// Prepare output file + ftruncate(out_fd, *file_size); + char * output = static_cast(mmap(nullptr, *file_size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_SHARED, out_fd, 0)); + if (output == reinterpret_cast(-1)) + { + perror(nullptr); + return 1; + } + + off_t size = 0; + off_t max_block_size = 1ull<<27; + + /// Compress data + while (in_offset < info_in.st_size) + { + size = ZSTD_findFrameCompressedSize(input + in_offset, max_block_size); + + /// Compress data or exit if error happens + if (0 != doDecompress(input, output, in_offset, out_offset, size, max_block_size, dctx)) + { + munmap(input, info_in.st_size); + munmap(output, *file_size); + return 1; + } + } + + /// Shrink file size and unmap + munmap(output, *file_size); + munmap(input, info_in.st_size); + return 0; +} int main() { - std::cout << "Test" << std::endl; + int input_fd = open("decompressor", O_RDONLY); + if (input_fd == -1) + { + perror(nullptr); + return 0; + } + + int output_fd = open("clickhouse_decompressed", O_RDWR | O_CREAT, 0775); + if (output_fd == -1) + { + perror(nullptr); + return 0; + } + + if (0 != decompress(input_fd, output_fd)) + { + return 1; + } + + close(input_fd); + close(output_fd); return 0; } diff --git a/utils/self-extr-exec/make_compression.sh b/utils/self-extr-exec/make_compression.sh new file mode 100755 index 00000000000..6fd8517c0db --- /dev/null +++ b/utils/self-extr-exec/make_compression.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +# Put this script near compressor and decompressor + +strip decompressor + +# TODO use env variables +./compressor ../../programs/clickhouse decompressor + \ No newline at end of file From 61212635d829c056402893510e8e50e044154992 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 30 Mar 2022 16:03:36 +0200 Subject: [PATCH 0023/1954] MergeTree multiple ORDER BY columns improve insert performance --- src/Columns/ColumnVector.cpp | 54 +++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index dded5ff6c99..82e2bba04d7 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -275,7 +275,59 @@ template void ColumnVector::updatePermutation(IColumn::PermutationSortDirection direction, IColumn::PermutationSortStability stability, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_ranges) const { - auto sort = [](auto begin, auto end, auto pred) { ::sort(begin, end, pred); }; + bool reverse = direction == IColumn::PermutationSortDirection::Descending; + bool ascending = direction == IColumn::PermutationSortDirection::Ascending; + bool sort_is_stable = stability == IColumn::PermutationSortStability::Stable; + + auto sort = [&](auto begin, auto end, auto pred) + { + /// A case for radix sort + if constexpr (is_arithmetic_v && !is_big_int_v) + { + /// TODO: LSD RadixSort is currently not stable if direction is descending, or value is floating point + bool use_radix_sort = (sort_is_stable && ascending && !std::is_floating_point_v) || !sort_is_stable; + size_t size = end - begin; + + /// Thresholds on size. Lower threshold is arbitrary. Upper threshold is chosen by the type for histogram counters. + if (size >= 256 && size <= std::numeric_limits::max() && use_radix_sort) + { + PaddedPODArray> pairs(size); + size_t index = 0; + + for (auto it = begin; it != end; ++it) + { + pairs[index] = {data[*it], static_cast(*it)}; + ++index; + } + + RadixSort>::executeLSD(pairs.data(), size, reverse, begin); + + /// Radix sort treats all NaNs to be greater than all numbers. + /// If the user needs the opposite, we must move them accordingly. + if (std::is_floating_point_v && nan_direction_hint < 0) + { + size_t nans_to_move = 0; + + for (size_t i = 0; i < size; ++i) + { + if (isNaN(data[begin[reverse ? i : size - 1 - i]])) + ++nans_to_move; + else + break; + } + + if (nans_to_move) + { + std::rotate(begin, begin + (reverse ? nans_to_move : size - nans_to_move), end); + } + } + + return; + } + } + + ::sort(begin, end, pred); + }; auto partial_sort = [](auto begin, auto mid, auto end, auto pred) { ::partial_sort(begin, mid, end, pred); }; if (direction == IColumn::PermutationSortDirection::Ascending && stability == IColumn::PermutationSortStability::Unstable) From 31e07fe078cee9410a1e13592cb473f5a5792afc Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 30 Mar 2022 16:53:25 +0300 Subject: [PATCH 0024/1954] Fixed crash when reporting successful login. --- src/Interpreters/SessionLog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp index a0c29c07d38..c808d7b66d7 100644 --- a/src/Interpreters/SessionLog.cpp +++ b/src/Interpreters/SessionLog.cpp @@ -212,8 +212,8 @@ void SessionLog::addLoginSuccess(const UUID & auth_id, std::optional ses DB::SessionLogElement log_entry(auth_id, SESSION_LOGIN_SUCCESS); log_entry.client_info = client_info; + if (const auto user = login_context.getUser()) { - const auto user = access->getUser(); log_entry.user = user->getName(); log_entry.user_identified_with = user->auth_data.getType(); log_entry.external_auth_server = user->auth_data.getLDAPServerName(); From dc88d8d5716c591a57741e6dc48e281aab2e7653 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 30 Mar 2022 23:37:24 +0300 Subject: [PATCH 0025/1954] Explicitly passing a user object to create a LogIn event --- src/Interpreters/Session.cpp | 3 ++- src/Interpreters/SessionLog.cpp | 11 ++++------- src/Interpreters/SessionLog.h | 3 ++- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 2af9a2b6bbc..7f71167834d 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -486,7 +486,8 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t session_log->addLoginSuccess( auth_id, named_session ? std::optional(named_session->key.second) : std::nullopt, - *query_context); + *query_context, + *user); notified_session_log_about_login = true; } diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp index c808d7b66d7..689f44f536a 100644 --- a/src/Interpreters/SessionLog.cpp +++ b/src/Interpreters/SessionLog.cpp @@ -203,7 +203,7 @@ void SessionLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insertData(auth_failure_reason.data(), auth_failure_reason.length()); } -void SessionLog::addLoginSuccess(const UUID & auth_id, std::optional session_id, const Context & login_context) +void SessionLog::addLoginSuccess(const UUID & auth_id, std::optional session_id, const Context & login_context, const User & login_user) { const auto access = login_context.getAccess(); const auto & settings = login_context.getSettingsRef(); @@ -212,12 +212,9 @@ void SessionLog::addLoginSuccess(const UUID & auth_id, std::optional ses DB::SessionLogElement log_entry(auth_id, SESSION_LOGIN_SUCCESS); log_entry.client_info = client_info; - if (const auto user = login_context.getUser()) - { - log_entry.user = user->getName(); - log_entry.user_identified_with = user->auth_data.getType(); - log_entry.external_auth_server = user->auth_data.getLDAPServerName(); - } + log_entry.user = login_user.getName(); + log_entry.user_identified_with = login_user.auth_data.getType(); + log_entry.external_auth_server = login_user.auth_data.getLDAPServerName(); if (session_id) log_entry.session_id = *session_id; diff --git a/src/Interpreters/SessionLog.h b/src/Interpreters/SessionLog.h index 26f137565cb..2c9ea1c69e3 100644 --- a/src/Interpreters/SessionLog.h +++ b/src/Interpreters/SessionLog.h @@ -18,6 +18,7 @@ enum SessionLogElementType : int8_t }; class ContextAccess; +struct User; /** A struct which will be inserted as row into session_log table. * @@ -69,7 +70,7 @@ class SessionLog : public SystemLog using SystemLog::SystemLog; public: - void addLoginSuccess(const UUID & auth_id, std::optional session_id, const Context & login_context); + void addLoginSuccess(const UUID & auth_id, std::optional session_id, const Context & login_context, const User & login_user); void addLoginFailure(const UUID & auth_id, const ClientInfo & info, const String & user, const Exception & reason); void addLogOut(const UUID & auth_id, const String & user, const ClientInfo & client_info); }; From 6c1d04710f556634cd8f53ad3bf994e51dd20965 Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Wed, 30 Mar 2022 22:01:50 +0000 Subject: [PATCH 0026/1954] finish self exctracting executable --- utils/self-extr-exec/compressor.cpp | 4 ++-- utils/self-extr-exec/decompressor.cpp | 15 ++++++++++----- utils/self-extr-exec/make_compression.sh | 5 +++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/utils/self-extr-exec/compressor.cpp b/utils/self-extr-exec/compressor.cpp index 951117ec8be..996a7080a6c 100644 --- a/utils/self-extr-exec/compressor.cpp +++ b/utils/self-extr-exec/compressor.cpp @@ -45,8 +45,8 @@ int compress(int in_fd, int out_fd, int level=5) // 6402520 is size of stripped decompressor size_t start = 6405000ull; - // 22550744 size of decompressor - // size_t start = 22550780ull; + // 22558008ull size of decompressor + // size_t start = 22558008ull; /// As experiments showed, size of compressed file is 4 times less than clickhouse executable /// Get a little bit more memory to prevent errors with size. diff --git a/utils/self-extr-exec/decompressor.cpp b/utils/self-extr-exec/decompressor.cpp index 6d6b44065ee..2c30342f28b 100644 --- a/utils/self-extr-exec/decompressor.cpp +++ b/utils/self-extr-exec/decompressor.cpp @@ -31,7 +31,7 @@ int decompress(int in_fd, int out_fd) fstat(in_fd, &info_in); /// NOTE: next parametrs depend on binary size - // 22550780ull for full, 6405000ull for stripped; + // 22558008ull for full, 6405000ull for stripped; off_t in_offset = 6405000ull /*size of decompressor*/, out_offset = 0; /// mmap files @@ -82,16 +82,16 @@ int decompress(int in_fd, int out_fd) return 0; } -int main() +int main(int /*argc*/, char* argv[]) { - int input_fd = open("decompressor", O_RDONLY); + int input_fd = open(argv[0], O_RDONLY); if (input_fd == -1) { perror(nullptr); return 0; } - int output_fd = open("clickhouse_decompressed", O_RDWR | O_CREAT, 0775); + int output_fd = open("clickhouse.decompressed", O_RDWR | O_CREAT, 0775); if (output_fd == -1) { perror(nullptr); @@ -103,7 +103,12 @@ int main() return 1; } - close(input_fd); + fsync(output_fd); close(output_fd); + close(input_fd); + + /// NOTE: This command should not depend from any variables. + /// It should be changed if file changes. + execl("/usr/bin/bash", "bash", "-c", "mv ./clickhouse.decompressed ./clickhouse", NULL); return 0; } diff --git a/utils/self-extr-exec/make_compression.sh b/utils/self-extr-exec/make_compression.sh index 6fd8517c0db..80d8ddb6519 100755 --- a/utils/self-extr-exec/make_compression.sh +++ b/utils/self-extr-exec/make_compression.sh @@ -2,8 +2,9 @@ # Put this script near compressor and decompressor -strip decompressor +cp decompressor clickhouse +strip clickhouse # TODO use env variables -./compressor ../../programs/clickhouse decompressor +./compressor ../../programs/clickhouse clickhouse \ No newline at end of file From 0df2302a6d0af46dce7d9ea542537cda02800f30 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 31 Mar 2022 21:16:51 +0300 Subject: [PATCH 0027/1954] Test that tries to reproduce a race condition --- .../02242_delete_user_race.reference | 0 .../0_stateless/02242_delete_user_race.sh | 78 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 tests/queries/0_stateless/02242_delete_user_race.reference create mode 100755 tests/queries/0_stateless/02242_delete_user_race.sh diff --git a/tests/queries/0_stateless/02242_delete_user_race.reference b/tests/queries/0_stateless/02242_delete_user_race.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02242_delete_user_race.sh b/tests/queries/0_stateless/02242_delete_user_race.sh new file mode 100755 index 00000000000..7b5bd366e69 --- /dev/null +++ b/tests/queries/0_stateless/02242_delete_user_race.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Tags: race, no-fasttest, no-parallel + +# Test tries to reproduce a race between threads: +# - deletes user +# - creates user +# - uses it as session user +# - apply role to the user +# +# https://github.com/ClickHouse/ClickHouse/issues/35714 + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh +# shellcheck source=./replication.lib +. "$CURDIR"/replication.lib + +#export PS4='\nDEBUG level:$SHLVL subshell-level: $BASH_SUBSHELL \nsource-file:${BASH_SOURCE} line#:${LINENO} function:${FUNCNAME[0]:+${FUNCNAME[0]}(): }\nstatement: ' + + +$CLICKHOUSE_CLIENT -nm -q " + DROP ROLE IF EXISTS test_role_02242; + CREATE ROLE test_role_02242; +" + +readonly REPEAT=1000 + +function delete_user() +{ + local i + for (( i = 0; i < REPEAT; ++i )) + do + $CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS test_user_02242" ||: + done +} + +function create_and_login_user() +{ + local i + for (( i = 0; i < REPEAT; ++i )) + do + $CLICKHOUSE_CLIENT -q "CREATE USER IF NOT EXISTS test_user_02242" ||: + $CLICKHOUSE_CLIENT -u "test_user_02242" -q "SELECT version()" > /dev/null ||: + done +} + +function set_role() +{ + local i + for (( i = 0; i < REPEAT; ++i )) + do + $CLICKHOUSE_CLIENT -q "SET ROLE test_role_02242 TO test_user_02242" ||: + done +} + +export -f delete_user +export -f create_and_login_user +export -f set_role + + +TIMEOUT=0.1 + +for (( i = 0 ; i < 1000; ++i )) +do + clickhouse_client_loop_timeout $TIMEOUT create_and_login_user 2> /dev/null & + clickhouse_client_loop_timeout $TIMEOUT delete_user 2> /dev/null & + clickhouse_client_loop_timeout $TIMEOUT login_user 2> /dev/null & + clickhouse_client_loop_timeout $TIMEOUT set_role 2> /dev/null & +done + +wait + +# $CLICKHOUSE_CLIENT -q "DROP ROLE IF EXISTS test_role_02242" +# $CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS test_user_02242" + +# wait From 183e952ac53461d297f7a1a310fbe8d30b00195a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 1 Apr 2022 03:52:42 +0200 Subject: [PATCH 0028/1954] Add clang-tidy --- cmake/analysis.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/analysis.cmake b/cmake/analysis.cmake index d1b9c86f15f..fd86e07dd6b 100644 --- a/cmake/analysis.cmake +++ b/cmake/analysis.cmake @@ -6,7 +6,7 @@ if (ENABLE_CLANG_TIDY) message(FATAL_ERROR "clang-tidy requires CMake version at least 3.6.") endif() - find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-13" "clang-tidy-12" "clang-tidy-11" "clang-tidy-10" "clang-tidy-9" "clang-tidy-8") + find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-14" "clang-tidy-13") if (CLANG_TIDY_PATH) message(STATUS From d89e03e04049ce1b128136b822507e2f775bca2d Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 1 Apr 2022 15:32:53 +0300 Subject: [PATCH 0029/1954] Made test less flakky --- tests/queries/0_stateless/02242_delete_user_race.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02242_delete_user_race.sh b/tests/queries/0_stateless/02242_delete_user_race.sh index 7b5bd366e69..256775e5334 100755 --- a/tests/queries/0_stateless/02242_delete_user_race.sh +++ b/tests/queries/0_stateless/02242_delete_user_race.sh @@ -62,7 +62,7 @@ export -f set_role TIMEOUT=0.1 -for (( i = 0 ; i < 1000; ++i )) +for (( i = 0 ; i < 100; ++i )) do clickhouse_client_loop_timeout $TIMEOUT create_and_login_user 2> /dev/null & clickhouse_client_loop_timeout $TIMEOUT delete_user 2> /dev/null & From 5a1392a8e349c2bba7e35b01b3e4f2b3c095aeae Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 5 Apr 2022 19:12:48 +0000 Subject: [PATCH 0030/1954] Try refactor something (1) --- .../getCompressionCodecForFile.cpp | 5 +- src/Compression/getCompressionCodecForFile.h | 3 +- .../MergeTree/DataPartStorageOnDisk.cpp | 99 +++++++++++++++ .../MergeTree/DataPartStorageOnDisk.h | 50 ++++++++ src/Storages/MergeTree/IDataPartStorage.h | 64 ++++++++++ src/Storages/MergeTree/IMergeTreeDataPart.cpp | 115 ++++++++---------- src/Storages/MergeTree/IMergeTreeDataPart.h | 11 +- src/Storages/MergeTree/MergeTreeData.cpp | 20 +-- src/Storages/MergeTree/MergeTreeData.h | 6 +- .../MergeTreeIndexGranularityInfo.cpp | 10 +- .../MergeTree/MergeTreeIndexGranularityInfo.h | 4 +- 11 files changed, 297 insertions(+), 90 deletions(-) create mode 100644 src/Storages/MergeTree/DataPartStorageOnDisk.cpp create mode 100644 src/Storages/MergeTree/DataPartStorageOnDisk.h create mode 100644 src/Storages/MergeTree/IDataPartStorage.h diff --git a/src/Compression/getCompressionCodecForFile.cpp b/src/Compression/getCompressionCodecForFile.cpp index a3ad2c93cc0..c88c62140f4 100644 --- a/src/Compression/getCompressionCodecForFile.cpp +++ b/src/Compression/getCompressionCodecForFile.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -12,9 +13,9 @@ namespace DB using Checksum = CityHash_v1_0_2::uint128; -CompressionCodecPtr getCompressionCodecForFile(const DiskPtr & disk, const String & relative_path) +CompressionCodecPtr getCompressionCodecForFile(const DataPartStoragePtr & data_part_storage, const String & relative_path) { - auto read_buffer = disk->readFile(relative_path); + auto read_buffer = data_part_storage->readFile(relative_path); read_buffer->ignore(sizeof(Checksum)); UInt8 header_size = ICompressionCodec::getHeaderSize(); diff --git a/src/Compression/getCompressionCodecForFile.h b/src/Compression/getCompressionCodecForFile.h index 4870de8b3b3..ad855684128 100644 --- a/src/Compression/getCompressionCodecForFile.h +++ b/src/Compression/getCompressionCodecForFile.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB { @@ -10,6 +11,6 @@ namespace DB /// clickhouse fashion (with checksums, headers for each block, etc). This /// method should be used as fallback when we cannot deduce compression codec /// from metadata. -CompressionCodecPtr getCompressionCodecForFile(const DiskPtr & disk, const String & relative_path); +CompressionCodecPtr getCompressionCodecForFile(const DataPartStoragePtr & data_part_storage, const String & relative_path); } diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp new file mode 100644 index 00000000000..073e8c2c6e0 --- /dev/null +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -0,0 +1,99 @@ +#include +#include +#include +#include + +namespace DB +{ + +DataPartStorageOnDisk::DataPartStorageOnDisk(VolumePtr volume_, std::string root_path_, std::string relative_root_path_) + : volume(std::move(volume_)), root_path(std::move(root_path_)), relative_root_path(std::move(relative_root_path_)) +{ +} + +std::unique_ptr DataPartStorageOnDisk::readFile( + const std::string & path, + const ReadSettings & settings, + std::optional read_hint, + std::optional file_size) const +{ + return volume->getDisk()->readFile(fs::path(relative_root_path) / path, settings, read_hint, file_size); +} + +bool DataPartStorageOnDisk::exists(const std::string & path) const +{ + return volume->getDisk()->exists(fs::path(relative_root_path) / path); +} + +bool DataPartStorageOnDisk::exists() const +{ + return volume->getDisk()->exists(relative_root_path); +} + +size_t DataPartStorageOnDisk::getFileSize(const String & path) const +{ + return volume->getDisk()->getFileSize(fs::path(relative_root_path) / path); +} + +DiskDirectoryIteratorPtr DataPartStorageOnDisk::iterate() const +{ + return volume->getDisk()->iterateDirectory(relative_root_path); +} + +DiskDirectoryIteratorPtr DataPartStorageOnDisk::iterateDirectory(const String & path) const +{ + return volume->getDisk()->iterateDirectory(fs::path(relative_root_path) / path); +} + +DataPartStoragePtr DataPartStorageOnDisk::getProjection(const std::string & name) const +{ + return std::make_shared(volume, fs::path(relative_root_path) / name); +} + +static UInt64 calculateTotalSizeOnDiskImpl(const DiskPtr & disk, const String & from) +{ + if (disk->isFile(from)) + return disk->getFileSize(from); + std::vector files; + disk->listFiles(from, files); + UInt64 res = 0; + for (const auto & file : files) + res += calculateTotalSizeOnDiskImpl(disk, fs::path(from) / file); + return res; +} + +UInt64 DataPartStorageOnDisk::calculateTotalSizeOnDisk() const +{ + return calculateTotalSizeOnDiskImpl(volume->getDisk(), relative_root_path); +} + +void DataPartStorageOnDisk::writeChecksums(MergeTreeDataPartChecksums & checksums) const +{ + std::string path = fs::path(relative_root_path) / "checksums.txt"; + + { + auto out = volume->getDisk()->writeFile(path + ".tmp", 4096); + checksums.write(*out); + } + + volume->getDisk()->moveFile(path + ".tmp", path); +} + +void DataPartStorageOnDisk::writeColumns(NamesAndTypesList & columns) const +{ + std::string path = fs::path(relative_root_path) / "columns.txt"; + + { + auto buf = volume->getDisk()->writeFile(path + ".tmp", 4096); + columns.writeText(*buf); + } + + volume->getDisk()->moveFile(path + ".tmp", path); +} + +std::string DataPartStorageOnDisk::getName() const +{ + return volume->getDisk()->getName(); +} + +} diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDisk.h new file mode 100644 index 00000000000..474833500c0 --- /dev/null +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.h @@ -0,0 +1,50 @@ +#pragma once +#include +#include +#include + +namespace DB +{ + +class IVolume; +using VolumePtr = std::shared_ptr; + + +class DataPartStorageOnDisk final : public IDataPartStorage +{ +public: + explicit DataPartStorageOnDisk(VolumePtr volume_, std::string root_path_, std::string relative_root_path_); + + std::unique_ptr readFile( + const std::string & path, + const ReadSettings & settings, + std::optional read_hint, + std::optional file_size) const override; + + bool exists(const std::string & path) const override; + bool exists() const override; + + size_t getFileSize(const std::string & path) const override; + + DiskDirectoryIteratorPtr iterate() const override; + DiskDirectoryIteratorPtr iterateDirectory(const std::string & path) const override; + + std::string getFullPath() const override { return root_path; } + std::string getFullRelativePath() const override { return relative_root_path; } + + UInt64 calculateTotalSizeOnDisk() const override; + + void writeChecksums(MergeTreeDataPartChecksums & checksums) const override; + void writeColumns(NamesAndTypesList & columns) const override; + + std::string getName() const override; + + DataPartStoragePtr getProjection(const std::string & name) const override; + +private: + VolumePtr volume; + std::string root_path; + std::string relative_root_path; +}; + +} diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h new file mode 100644 index 00000000000..1bf5c890ee8 --- /dev/null +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -0,0 +1,64 @@ +#pragma once +#include +#include +#include +#include + +namespace DB +{ + +class ReadBufferFromFileBase; + + +class IDiskDirectoryIterator; +using DiskDirectoryIteratorPtr = std::unique_ptr; + +struct MergeTreeDataPartChecksums; + +/// This is an abstraction of storage for data part files. +/// Generally, it contains read-only methods from IDisk. +class IDataPartStorage +{ +private: +public: + virtual ~IDataPartStorage() = default; + + /// Open the file for read and return ReadBufferFromFileBase object. + virtual std::unique_ptr readFile( + const std::string & path, + const ReadSettings & settings, + std::optional read_hint, + std::optional file_size) const = 0; + + virtual bool exists(const std::string & path) const = 0; + virtual bool exists() const = 0; + + virtual DiskDirectoryIteratorPtr iterate() const = 0; + virtual DiskDirectoryIteratorPtr iterateDirectory(const std::string & path) const = 0; + + virtual size_t getFileSize(const std::string & path) const = 0; + + virtual std::string getFullPath() const = 0; + virtual std::string getFullRelativePath() const = 0; + + virtual UInt64 calculateTotalSizeOnDisk() const = 0; + + /// Should remove it later + virtual void writeChecksums(MergeTreeDataPartChecksums & checksums) const = 0; + virtual void writeColumns(NamesAndTypesList & columns) const = 0; + + /// Disk name + virtual std::string getName() const = 0; + + virtual std::shared_ptr getProjection(const std::string & name) const = 0; +}; + +class IDataPartStorageBuilder +{ +public: + virtual ~IDataPartStorageBuilder() = default; +}; + +using DataPartStoragePtr = std::shared_ptr; + +} diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 0ed96f5dda4..5c944d284d6 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -298,14 +298,14 @@ static void decrementTypeMetric(MergeTreeDataPartType type) IMergeTreeDataPart::IMergeTreeDataPart( const MergeTreeData & storage_, const String & name_, - const VolumePtr & volume_, + const DataPartStoragePtr & data_part_storage_, const std::optional & relative_path_, Type part_type_, const IMergeTreeDataPart * parent_part_) : storage(storage_) , name(name_) , info(MergeTreePartInfo::fromPartName(name_, storage.format_version)) - , volume(parent_part_ ? parent_part_->volume : volume_) + , data_part_storage(parent_part_ ? parent_part_->data_part_storage : data_part_storage_) , relative_path(relative_path_.value_or(name_)) , index_granularity_info(storage_, part_type_) , part_type(part_type_) @@ -326,14 +326,14 @@ IMergeTreeDataPart::IMergeTreeDataPart( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const VolumePtr & volume_, + const DataPartStoragePtr & data_part_storage_, const std::optional & relative_path_, Type part_type_, const IMergeTreeDataPart * parent_part_) : storage(storage_) , name(name_) , info(info_) - , volume(parent_part_ ? parent_part_->volume : volume_) + , data_part_storage(parent_part_ ? parent_part_->data_part_storage : data_part_storage_) , relative_path(relative_path_.value_or(name_)) , index_granularity_info(storage_, part_type_) , part_type(part_type_) @@ -471,9 +471,9 @@ void IMergeTreeDataPart::removeIfNeeded() try { - auto path = getFullRelativePath(); + auto path = data_part_storage->getFullRelativePath(); - if (!volume->getDisk()->exists(path)) + if (!data_part_storage->exists()) // path return; if (is_temp) @@ -605,26 +605,26 @@ String IMergeTreeDataPart::getColumnNameWithMinimumCompressedSize(const StorageS } if (!minimum_size_column) - throw Exception("Could not find a column of minimum size in MergeTree, part " + getFullPath(), ErrorCodes::LOGICAL_ERROR); + throw Exception("Could not find a column of minimum size in MergeTree, part " + data_part_storage->getFullPath(), ErrorCodes::LOGICAL_ERROR); return *minimum_size_column; } -String IMergeTreeDataPart::getFullPath() const -{ - if (relative_path.empty()) - throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); +// String IMergeTreeDataPart::getFullPath() const +// { +// if (relative_path.empty()) +// throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); - return fs::path(storage.getFullPathOnDisk(volume->getDisk())) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; -} +// return fs::path(storage.getFullPathOnDisk(volume->getDisk())) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; +// } -String IMergeTreeDataPart::getFullRelativePath() const -{ - if (relative_path.empty()) - throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); +// String IMergeTreeDataPart::getFullRelativePath() const +// { +// if (relative_path.empty()) +// throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); - return fs::path(storage.relative_data_path) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; -} +// return fs::path(storage.relative_data_path) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; +// } void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) { @@ -698,10 +698,11 @@ void IMergeTreeDataPart::loadProjections(bool require_columns_checksums, bool ch auto metadata_snapshot = storage.getInMemoryMetadataPtr(); for (const auto & projection : metadata_snapshot->projections) { - String path = getFullRelativePath() + projection.name + ".proj"; - if (volume->getDisk()->exists(path)) + String path = /*getFullRelativePath() + */ projection.name + ".proj"; + if (data_part_storage->exists(path)) { - auto part = storage.createPart(projection.name, {"all", 0, 0, 0}, volume, projection.name + ".proj", this); + auto projection_part_storage = data_part_storage->getProjection(projection.name + ".proj"); + auto part = storage.createPart(projection.name, {"all", 0, 0, 0}, projection_part_storage, this); part->loadColumnsChecksumsIndexes(require_columns_checksums, check_consistency); projection_parts.emplace(projection.name, std::move(part)); } @@ -742,7 +743,7 @@ void IMergeTreeDataPart::loadIndex() } String index_name = "primary.idx"; - String index_path = fs::path(getFullRelativePath()) / index_name; + String index_path = fs::path(data_part_storage->getFullRelativePath()) / index_name; auto index_file = metadata_manager->read(index_name); size_t marks_count = index_granularity.getMarksCount(); @@ -764,7 +765,7 @@ void IMergeTreeDataPart::loadIndex() } if (!index_file->eof()) - throw Exception("Index file " + fullPath(volume->getDisk(), index_path) + " is unexpectedly long", ErrorCodes::EXPECTED_END_OF_FILE); + throw Exception("Index file " + index_path + " is unexpectedly long", ErrorCodes::EXPECTED_END_OF_FILE); index.assign(std::make_move_iterator(loaded_index.begin()), std::make_move_iterator(loaded_index.end())); } @@ -789,9 +790,9 @@ NameSet IMergeTreeDataPart::getFileNamesWithoutChecksums() const return {}; NameSet result = {"checksums.txt", "columns.txt"}; - String default_codec_path = fs::path(getFullRelativePath()) / DEFAULT_COMPRESSION_CODEC_FILE_NAME; + //String default_codec_path = fs::path(getFullRelativePath()) / DEFAULT_COMPRESSION_CODEC_FILE_NAME; - if (volume->getDisk()->exists(default_codec_path)) + if (data_part_storage->exists(DEFAULT_COMPRESSION_CODEC_FILE_NAME)) result.emplace(DEFAULT_COMPRESSION_CODEC_FILE_NAME); return result; @@ -806,7 +807,7 @@ void IMergeTreeDataPart::loadDefaultCompressionCodec() return; } - String path = fs::path(getFullRelativePath()) / DEFAULT_COMPRESSION_CODEC_FILE_NAME; + String path = fs::path(data_part_storage->getFullRelativePath()) / DEFAULT_COMPRESSION_CODEC_FILE_NAME; bool exists = metadata_manager->exists(DEFAULT_COMPRESSION_CODEC_FILE_NAME); if (!exists) { @@ -872,10 +873,10 @@ CompressionCodecPtr IMergeTreeDataPart::detectDefaultCompressionCodec() const { if (path_to_data_file.empty()) { - String candidate_path = fs::path(getFullRelativePath()) / (ISerialization::getFileNameForStream(part_column, substream_path) + ".bin"); + String candidate_path = /*fs::path(getFullRelativePath()) */ (ISerialization::getFileNameForStream(part_column, substream_path) + ".bin"); /// We can have existing, but empty .bin files. Example: LowCardinality(Nullable(...)) columns and column_name.dict.null.bin file. - if (volume->getDisk()->exists(candidate_path) && volume->getDisk()->getFileSize(candidate_path) != 0) + if (data_part_storage->exists(candidate_path) && data_part_storage->getFileSize(candidate_path) != 0) path_to_data_file = candidate_path; } }); @@ -886,7 +887,7 @@ CompressionCodecPtr IMergeTreeDataPart::detectDefaultCompressionCodec() const continue; } - result = getCompressionCodecForFile(volume->getDisk(), path_to_data_file); + result = getCompressionCodecForFile(data_part_storage, path_to_data_file); break; } } @@ -911,7 +912,7 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() } else { - String path = getFullRelativePath(); + //String path = getFullRelativePath(); if (!parent_part) partition.load(storage, metadata_manager); @@ -931,7 +932,7 @@ void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() String calculated_partition_id = partition.getID(metadata_snapshot->getPartitionKey().sample_block); if (calculated_partition_id != info.partition_id) throw Exception( - "While loading part " + getFullPath() + ": calculated partition ID: " + calculated_partition_id + "While loading part " + data_part_storage->getFullPath() + ": calculated partition ID: " + calculated_partition_id + " differs from partition ID in part name: " + info.partition_id, ErrorCodes::CORRUPTED_DATA); } @@ -951,7 +952,7 @@ void IMergeTreeDataPart::appendFilesOfPartitionAndMinMaxIndex(Strings & files) c void IMergeTreeDataPart::loadChecksums(bool require) { - const String path = fs::path(getFullRelativePath()) / "checksums.txt"; + //const String path = fs::path(getFullRelativePath()) / "checksums.txt"; bool exists = metadata_manager->exists("checksums.txt"); if (exists) { @@ -962,7 +963,7 @@ void IMergeTreeDataPart::loadChecksums(bool require) bytes_on_disk = checksums.getTotalSizeOnDisk(); } else - bytes_on_disk = calculateTotalSizeOnDisk(volume->getDisk(), getFullRelativePath()); + bytes_on_disk = data_part_storage->calculateTotalSizeOnDisk(); //calculateTotalSizeOnDisk(volume->getDisk(), getFullRelativePath()); } else { @@ -974,13 +975,7 @@ void IMergeTreeDataPart::loadChecksums(bool require) LOG_WARNING(storage.log, "Checksums for part {} not found. Will calculate them from data on disk.", name); checksums = checkDataPart(shared_from_this(), false); - - { - auto out = volume->getDisk()->writeFile(fs::path(getFullRelativePath()) / "checksums.txt.tmp", 4096); - checksums.write(*out); - } - - volume->getDisk()->moveFile(fs::path(getFullRelativePath()) / "checksums.txt.tmp", fs::path(getFullRelativePath()) / "checksums.txt"); + data_part_storage->writeChecksums(checksums); bytes_on_disk = checksums.getTotalSizeOnDisk(); } @@ -993,7 +988,7 @@ void IMergeTreeDataPart::appendFilesOfChecksums(Strings & files) void IMergeTreeDataPart::loadRowsCount() { - String path = fs::path(getFullRelativePath()) / "count.txt"; + //String path = fs::path(getFullRelativePath()) / "count.txt"; auto read_rows_count = [&]() { @@ -1065,7 +1060,7 @@ void IMergeTreeDataPart::loadRowsCount() } else { - if (volume->getDisk()->exists(path)) + if (data_part_storage->exists("count.txt")) { read_rows_count(); return; @@ -1164,7 +1159,7 @@ void IMergeTreeDataPart::appendFilesOfUUID(Strings & files) void IMergeTreeDataPart::loadColumns(bool require) { - String path = fs::path(getFullRelativePath()) / "columns.txt"; + String path = fs::path(data_part_storage->getFullRelativePath()) / "columns.txt"; auto metadata_snapshot = storage.getInMemoryMetadataPtr(); if (parent_part) metadata_snapshot = metadata_snapshot->projections.get(name).metadata; @@ -1175,22 +1170,18 @@ void IMergeTreeDataPart::loadColumns(bool require) { /// We can get list of columns only from columns.txt in compact parts. if (require || part_type == Type::COMPACT) - throw Exception("No columns.txt in part " + name + ", expected path " + path + " on drive " + volume->getDisk()->getName(), + throw Exception("No columns.txt in part " + name + ", expected path " + path + " on drive " + data_part_storage->getName(), ErrorCodes::NO_FILE_IN_DATA_PART); /// If there is no file with a list of columns, write it down. for (const NameAndTypePair & column : metadata_snapshot->getColumns().getAllPhysical()) - if (volume->getDisk()->exists(fs::path(getFullRelativePath()) / (getFileNameForColumn(column) + ".bin"))) + if (data_part_storage->exists(getFileNameForColumn(column) + ".bin")) loaded_columns.push_back(column); if (columns.empty()) throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - { - auto buf = volume->getDisk()->writeFile(path + ".tmp", 4096); - loaded_columns.writeText(*buf); - } - volume->getDisk()->moveFile(path + ".tmp", path); + data_part_storage->writeColumns(loaded_columns); } else { @@ -1239,17 +1230,17 @@ bool IMergeTreeDataPart::shallParticipateInMerges(const StoragePolicyPtr & stora return !volume_ptr->areMergesAvoided(); } -UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const DiskPtr & disk_, const String & from) -{ - if (disk_->isFile(from)) - return disk_->getFileSize(from); - std::vector files; - disk_->listFiles(from, files); - UInt64 res = 0; - for (const auto & file : files) - res += calculateTotalSizeOnDisk(disk_, fs::path(from) / file); - return res; -} +// UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const DataPartStoragePtr & data_part_storage_, const String & from) +// { +// if (data_part_storage_->isFile(from)) +// return data_part_storage_->getFileSize(from); +// std::vector files; +// disk_->listFiles(from, files); +// UInt64 res = 0; +// for (const auto & file : files) +// res += calculateTotalSizeOnDisk(data_part_storage_, fs::path(from) / file); +// return res; +// } void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_new_dir_if_exists) const diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 694be50d168..b483d694220 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -68,7 +69,7 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const VolumePtr & volume, + const DataPartStoragePtr & data_part_storage_, const std::optional & relative_path, Type part_type_, const IMergeTreeDataPart * parent_part_); @@ -76,7 +77,7 @@ public: IMergeTreeDataPart( const MergeTreeData & storage_, const String & name_, - const VolumePtr & volume, + const DataPartStoragePtr & data_part_storage_, const std::optional & relative_path, Type part_type_, const IMergeTreeDataPart * parent_part_); @@ -194,7 +195,7 @@ public: /// processed by multiple shards. UUID uuid = UUIDHelpers::Nil; - VolumePtr volume; + DataPartStoragePtr data_part_storage; /// A directory path (relative to storage's path) where part data is actually stored /// Examples: 'detached/tmp_fetch_', 'tmp_', '' @@ -338,10 +339,10 @@ public: size_t getFileSizeOrZero(const String & file_name) const; /// Returns path to part dir relatively to disk mount point - String getFullRelativePath() const; + // String getFullRelativePath() const; /// Returns full path to part dir - String getFullPath() const; + // String getFullPath() const; /// Moves a part to detached/ directory and adds prefix to its name void renameToDetached(const String & prefix) const; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index e4868cb00fb..6c4ef0ce6fa 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2273,14 +2273,14 @@ MergeTreeDataPartType MergeTreeData::choosePartTypeOnDisk(size_t bytes_uncompres MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, MergeTreeDataPartType type, const MergeTreePartInfo & part_info, - const VolumePtr & volume, const String & relative_path, const IMergeTreeDataPart * parent_part) const + const DataPartStoragePtr & data_part_storage, const String & relative_path, const IMergeTreeDataPart * parent_part) const { if (type == MergeTreeDataPartType::COMPACT) - return std::make_shared(*this, name, part_info, volume, relative_path, parent_part); + return std::make_shared(*this, name, part_info, data_part_storage, parent_part); else if (type == MergeTreeDataPartType::WIDE) - return std::make_shared(*this, name, part_info, volume, relative_path, parent_part); + return std::make_shared(*this, name, part_info, data_part_storage, parent_part); else if (type == MergeTreeDataPartType::IN_MEMORY) - return std::make_shared(*this, name, part_info, volume, relative_path, parent_part); + return std::make_shared(*this, name, part_info, data_part_storage, parent_part); else throw Exception("Unknown type of part " + relative_path, ErrorCodes::UNKNOWN_PART_TYPE); } @@ -2298,18 +2298,18 @@ static MergeTreeDataPartType getPartTypeFromMarkExtension(const String & mrk_ext } MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( - const String & name, const VolumePtr & volume, const String & relative_path, const IMergeTreeDataPart * parent_part) const + const String & name, const DataPartStoragePtr & data_part_storage, const String & relative_path, const IMergeTreeDataPart * parent_part) const { - return createPart(name, MergeTreePartInfo::fromPartName(name, format_version), volume, relative_path, parent_part); + return createPart(name, MergeTreePartInfo::fromPartName(name, format_version), data_part_storage, relative_path, parent_part); } MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( const String & name, const MergeTreePartInfo & part_info, - const VolumePtr & volume, const String & relative_path, const IMergeTreeDataPart * parent_part) const + const DataPartStoragePtr & data_part_storage, const String & relative_path, const IMergeTreeDataPart * parent_part) const { MergeTreeDataPartType type; - auto full_path = fs::path(relative_data_path) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; - auto mrk_ext = MergeTreeIndexGranularityInfo::getMarksExtensionFromFilesystem(volume->getDisk(), full_path); + // auto full_path = fs::path(relative_data_path) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; + auto mrk_ext = MergeTreeIndexGranularityInfo::getMarksExtensionFromFilesystem(data_part_storage); if (mrk_ext) type = getPartTypeFromMarkExtension(*mrk_ext); @@ -2319,7 +2319,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( type = choosePartTypeOnDisk(0, 0); } - return createPart(name, type, part_info, volume, relative_path, parent_part); + return createPart(name, type, part_info, data_part_storage, relative_path, parent_part); } void MergeTreeData::changeSettings( diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 1cbcd4282d0..23b1cecb32a 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -229,15 +229,15 @@ public: /// After this method setColumns must be called MutableDataPartPtr createPart(const String & name, MergeTreeDataPartType type, const MergeTreePartInfo & part_info, - const VolumePtr & volume, const String & relative_path, const IMergeTreeDataPart * parent_part = nullptr) const; + const DataPartStoragePtr & data_part_storage, const IMergeTreeDataPart * parent_part = nullptr) const; /// Create part, that already exists on filesystem. /// After this methods 'loadColumnsChecksumsIndexes' must be called. MutableDataPartPtr createPart(const String & name, - const VolumePtr & volume, const String & relative_path, const IMergeTreeDataPart * parent_part = nullptr) const; + const DataPartStoragePtr & data_part_storage, const IMergeTreeDataPart * parent_part = nullptr) const; MutableDataPartPtr createPart(const String & name, const MergeTreePartInfo & part_info, - const VolumePtr & volume, const String & relative_path, const IMergeTreeDataPart * parent_part = nullptr) const; + const DataPartStoragePtr & data_part_storage, const IMergeTreeDataPart * parent_part = nullptr) const; /// Auxiliary object to add a set of parts into the working set in two steps: /// * First, as PreActive parts (the parts are ready, but not yet in the active set). diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index 6da0a822f7f..e8a8843bee1 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -13,11 +13,11 @@ namespace ErrorCodes extern const int UNKNOWN_PART_TYPE; } -std::optional MergeTreeIndexGranularityInfo::getMarksExtensionFromFilesystem(const DiskPtr & disk, const String & path_to_part) +std::optional MergeTreeIndexGranularityInfo::getMarksExtensionFromFilesystem(const DataPartStoragePtr & data_part_storage) { - if (disk->exists(path_to_part)) + if (data_part_storage->exists()) { - for (DiskDirectoryIteratorPtr it = disk->iterateDirectory(path_to_part); it->isValid(); it->next()) + for (DiskDirectoryIteratorPtr it = data_part_storage->iterate(); it->isValid(); it->next()) { const auto & ext = fs::path(it->path()).extension(); if (ext == getNonAdaptiveMrkExtension() @@ -46,9 +46,9 @@ MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo(const MergeTreeData setAdaptive(storage_settings->index_granularity_bytes); } -void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const DiskPtr & disk, const String & path_to_part) +void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const DataPartStoragePtr & data_part_storage) { - auto mrk_ext = getMarksExtensionFromFilesystem(disk, path_to_part); + auto mrk_ext = getMarksExtensionFromFilesystem(data_part_storage); if (mrk_ext && *mrk_ext == getNonAdaptiveMrkExtension()) setNonAdaptive(); } diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index 92e889ec762..1893c25e22c 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -28,7 +28,7 @@ public: MergeTreeIndexGranularityInfo(const MergeTreeData & storage, MergeTreeDataPartType type_); - void changeGranularityIfRequired(const DiskPtr & disk, const String & path_to_part); + void changeGranularityIfRequired(const DataPartStoragePtr & data_part_storage); String getMarksFilePath(const String & path_prefix) const { @@ -37,7 +37,7 @@ public: size_t getMarkSizeInBytes(size_t columns_num = 1) const; - static std::optional getMarksExtensionFromFilesystem(const DiskPtr & disk, const String & path_to_part); + static std::optional getMarksExtensionFromFilesystem(const DataPartStoragePtr & data_part_storage); private: MergeTreeDataPartType type; From 5cbec379078a5ba927ee5dfce94c07564a4a0f6c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 7 Apr 2022 11:58:38 +0000 Subject: [PATCH 0031/1954] Part 2 --- .../MergeTree/DataPartStorageOnDisk.cpp | 86 ++++++++++++++++--- .../MergeTree/DataPartStorageOnDisk.h | 11 ++- src/Storages/MergeTree/IDataPartStorage.h | 7 ++ src/Storages/MergeTree/IMergeTreeDataPart.cpp | 13 ++- src/Storages/MergeTree/IPartMetadataManager.h | 2 + .../MergeTree/PartMetadataManagerWithCache.h | 6 ++ 6 files changed, 107 insertions(+), 18 deletions(-) diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 073e8c2c6e0..0ec9acaa0c6 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -2,52 +2,65 @@ #include #include #include +#include +#include namespace DB { -DataPartStorageOnDisk::DataPartStorageOnDisk(VolumePtr volume_, std::string root_path_, std::string relative_root_path_) - : volume(std::move(volume_)), root_path(std::move(root_path_)), relative_root_path(std::move(relative_root_path_)) +namespace ErrorCodes +{ + extern const int FILE_DOESNT_EXIST; + extern const int DIRECTORY_ALREADY_EXISTS; +} + +DataPartStorageOnDisk::DataPartStorageOnDisk(VolumePtr volume_, std::string root_path_) + : volume(std::move(volume_)), root_path(std::move(root_path_)) { } +std::string DataPartStorageOnDisk::getFullPath() const +{ + return fs::path(volume->getDisk()->getPath()) / root_path; +} + std::unique_ptr DataPartStorageOnDisk::readFile( const std::string & path, const ReadSettings & settings, std::optional read_hint, std::optional file_size) const { - return volume->getDisk()->readFile(fs::path(relative_root_path) / path, settings, read_hint, file_size); + return volume->getDisk()->readFile(fs::path(root_path) / path, settings, read_hint, file_size); } bool DataPartStorageOnDisk::exists(const std::string & path) const { - return volume->getDisk()->exists(fs::path(relative_root_path) / path); + return volume->getDisk()->exists(fs::path(root_path) / path); } bool DataPartStorageOnDisk::exists() const { - return volume->getDisk()->exists(relative_root_path); + return volume->getDisk()->exists(root_path); } size_t DataPartStorageOnDisk::getFileSize(const String & path) const { - return volume->getDisk()->getFileSize(fs::path(relative_root_path) / path); + return volume->getDisk()->getFileSize(fs::path(root_path) / path); } DiskDirectoryIteratorPtr DataPartStorageOnDisk::iterate() const { - return volume->getDisk()->iterateDirectory(relative_root_path); + return volume->getDisk()->iterateDirectory(root_path); } DiskDirectoryIteratorPtr DataPartStorageOnDisk::iterateDirectory(const String & path) const { - return volume->getDisk()->iterateDirectory(fs::path(relative_root_path) / path); + return volume->getDisk()->iterateDirectory(fs::path(root_path) / path); } DataPartStoragePtr DataPartStorageOnDisk::getProjection(const std::string & name) const { - return std::make_shared(volume, fs::path(relative_root_path) / name); + return std::make_shared(volume, fs::path(root_path) / name); } static UInt64 calculateTotalSizeOnDiskImpl(const DiskPtr & disk, const String & from) @@ -64,12 +77,12 @@ static UInt64 calculateTotalSizeOnDiskImpl(const DiskPtr & disk, const String & UInt64 DataPartStorageOnDisk::calculateTotalSizeOnDisk() const { - return calculateTotalSizeOnDiskImpl(volume->getDisk(), relative_root_path); + return calculateTotalSizeOnDiskImpl(volume->getDisk(), root_path); } void DataPartStorageOnDisk::writeChecksums(MergeTreeDataPartChecksums & checksums) const { - std::string path = fs::path(relative_root_path) / "checksums.txt"; + std::string path = fs::path(root_path) / "checksums.txt"; { auto out = volume->getDisk()->writeFile(path + ".tmp", 4096); @@ -81,7 +94,7 @@ void DataPartStorageOnDisk::writeChecksums(MergeTreeDataPartChecksums & checksum void DataPartStorageOnDisk::writeColumns(NamesAndTypesList & columns) const { - std::string path = fs::path(relative_root_path) / "columns.txt"; + std::string path = fs::path(root_path) / "columns.txt"; { auto buf = volume->getDisk()->writeFile(path + ".tmp", 4096); @@ -91,6 +104,55 @@ void DataPartStorageOnDisk::writeColumns(NamesAndTypesList & columns) const volume->getDisk()->moveFile(path + ".tmp", path); } +void DataPartStorageOnDisk::rename(const String & new_relative_path, Poco::Logger * log, bool remove_new_dir_if_exists, bool fsync) +{ + if (!volume->getDisk()->exists(root_path)) + throw Exception("Part directory " + fullPath(volume->getDisk(), root_path) + " doesn't exist. Most likely it is a logical error.", ErrorCodes::FILE_DOESNT_EXIST); + + /// Why? + String to = fs::path(new_relative_path) / ""; + + if (volume->getDisk()->exists(to)) + { + if (remove_new_dir_if_exists) + { + Names files; + volume->getDisk()->listFiles(to, files); + + LOG_WARNING(log, "Part directory {} already exists and contains {} files. Removing it.", fullPath(volume->getDisk(), to), files.size()); + + volume->getDisk()->removeRecursive(to); + } + else + { + throw Exception("Part directory " + fullPath(volume->getDisk(), to) + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); + } + } + + // metadata_manager->deleteAll(true); + // metadata_manager->assertAllDeleted(true); + + /// Why? + volume->getDisk()->setLastModified(root_path, Poco::Timestamp::fromEpochTime(time(nullptr))); + volume->getDisk()->moveDirectory(root_path, to); + root_path = new_relative_path; + // metadata_manager->updateAll(true); + + SyncGuardPtr sync_guard; + if (fsync) + sync_guard = volume->getDisk()->getDirectorySyncGuard(root_path); +} + +bool DataPartStorageOnDisk::shallParticipateInMerges(const IStoragePolicy & storage_policy) const +{ + /// `IMergeTreeDataPart::volume` describes space where current part belongs, and holds + /// `SingleDiskVolume` object which does not contain up-to-date settings of corresponding volume. + /// Therefore we shall obtain volume from storage policy. + auto volume_ptr = storage_policy.getVolume(storage_policy.getVolumeIndexByDisk(volume->getDisk())); + + return !volume_ptr->areMergesAvoided(); +} + std::string DataPartStorageOnDisk::getName() const { return volume->getDisk()->getName(); diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDisk.h index 474833500c0..69858440fe1 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.h +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.h @@ -13,7 +13,7 @@ using VolumePtr = std::shared_ptr; class DataPartStorageOnDisk final : public IDataPartStorage { public: - explicit DataPartStorageOnDisk(VolumePtr volume_, std::string root_path_, std::string relative_root_path_); + explicit DataPartStorageOnDisk(VolumePtr volume_, std::string root_path_); std::unique_ptr readFile( const std::string & path, @@ -29,14 +29,18 @@ public: DiskDirectoryIteratorPtr iterate() const override; DiskDirectoryIteratorPtr iterateDirectory(const std::string & path) const override; - std::string getFullPath() const override { return root_path; } - std::string getFullRelativePath() const override { return relative_root_path; } + std::string getFullPath() const override; + std::string getFullRelativePath() const override { return root_path; } UInt64 calculateTotalSizeOnDisk() const override; void writeChecksums(MergeTreeDataPartChecksums & checksums) const override; void writeColumns(NamesAndTypesList & columns) const override; + bool shallParticipateInMerges(const IStoragePolicy &) const; + + void rename(const String & new_relative_path, Poco::Logger * log, bool remove_new_dir_if_exists, bool fsync) override; + std::string getName() const override; DataPartStoragePtr getProjection(const std::string & name) const override; @@ -44,7 +48,6 @@ public: private: VolumePtr volume; std::string root_path; - std::string relative_root_path; }; } diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 1bf5c890ee8..5af628971bf 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -15,6 +15,8 @@ using DiskDirectoryIteratorPtr = std::unique_ptr; struct MergeTreeDataPartChecksums; +class IStoragePolicy; + /// This is an abstraction of storage for data part files. /// Generally, it contains read-only methods from IDisk. class IDataPartStorage @@ -47,6 +49,11 @@ public: virtual void writeChecksums(MergeTreeDataPartChecksums & checksums) const = 0; virtual void writeColumns(NamesAndTypesList & columns) const = 0; + /// A leak of abstraction + virtual bool shallParticipateInMerges(const IStoragePolicy &) const { return true; } + + virtual void rename(const String & new_relative_path, Poco::Logger * log, bool remove_new_dir_if_exists, bool fsync); + /// Disk name virtual std::string getName() const = 0; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 5c944d284d6..fc66f547565 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1248,8 +1248,15 @@ try { assertOnDisk(); - String from = getFullRelativePath(); - String to = fs::path(storage.relative_data_path) / (parent_part ? parent_part->relative_path : "") / new_relative_path / ""; + if (parent_part) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Move is not supported for projection parts: moving form {} to {}", + data_part_storage->getFullPath(), new_relative_path); + + String to = fs::path(storage.relative_data_path) / new_relative_path / ""; + + data_part_storage->move(to); if (!volume->getDisk()->exists(from)) throw Exception("Part directory " + fullPath(volume->getDisk(), from) + " doesn't exist. Most likely it is a logical error.", ErrorCodes::FILE_DOESNT_EXIST); @@ -1278,6 +1285,8 @@ try relative_path = new_relative_path; metadata_manager->updateAll(true); + metadata_manager->move(from, to); + SyncGuardPtr sync_guard; if (storage.getSettings()->fsync_part_directory) sync_guard = volume->getDisk()->getDirectorySyncGuard(to); diff --git a/src/Storages/MergeTree/IPartMetadataManager.h b/src/Storages/MergeTree/IPartMetadataManager.h index 876000de412..68bdada3464 100644 --- a/src/Storages/MergeTree/IPartMetadataManager.h +++ b/src/Storages/MergeTree/IPartMetadataManager.h @@ -47,6 +47,8 @@ public: /// If include_projection is true, also update metadatas in projection parts. virtual void updateAll(bool include_projection) = 0; + virtual void move(const String & from, const String & to) = 0; + /// Check all metadatas in part. virtual std::unordered_map check() const = 0; diff --git a/src/Storages/MergeTree/PartMetadataManagerWithCache.h b/src/Storages/MergeTree/PartMetadataManagerWithCache.h index 06e7a85ba2b..856ea991611 100644 --- a/src/Storages/MergeTree/PartMetadataManagerWithCache.h +++ b/src/Storages/MergeTree/PartMetadataManagerWithCache.h @@ -34,6 +34,8 @@ public: /// Need to be called after part directory is renamed. void updateAll(bool include_projection) override; + void move(const String & from, const String & to) override; + /// Check if all metadatas in part from RocksDB cache are up to date. std::unordered_map check() const override; @@ -49,6 +51,10 @@ private: void getKeysAndCheckSums(Strings & keys, std::vector & checksums) const; + void deleteAllImpl(const String & path, bool include_projection); + void assertAllDeletedImpl(const String & path, bool include_projection) const; + void updateAllImpl(const String & path, bool include_projection); + MergeTreeMetadataCachePtr cache; }; From 3ef9036f52e86f251f277008a671eb303017d688 Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Thu, 7 Apr 2022 12:21:14 +0000 Subject: [PATCH 0032/1954] improve compressor --- utils/self-extr-exec/compressor.cpp | 323 ++++++++++++++++++++------ utils/self-extr-exec/decompressor.cpp | 272 ++++++++++++++++++---- 2 files changed, 480 insertions(+), 115 deletions(-) diff --git a/utils/self-extr-exec/compressor.cpp b/utils/self-extr-exec/compressor.cpp index 996a7080a6c..4a067f45890 100644 --- a/utils/self-extr-exec/compressor.cpp +++ b/utils/self-extr-exec/compressor.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -7,6 +8,47 @@ #include #include +/* +Overview of compression: + ______________________ + | Decompressor | + |----------------------| + | Compressed file 1 | + | Compressed file 2 | + | ... | + |----------------------| + | Info about 1 file | + | Info about 2 file | + | ... | + |----------------------| + | Metadata | + |______________________| +*/ + +/* +Metadata contains: + 1) number of files to support multiple file compression + 2) start_of_files_data to know start of files metadata + 3) end of binary to know start of compressed data + 4) uncompressed data size +*/ +struct MetaData +{ + size_t number_of_files = 0; + size_t start_of_files_data = 0; +}; + +/// Information about each file for correct extraction. +/// Each file data is followed by name of file +/// with length equals to name_length. +struct FileData +{ + size_t start = 0; + size_t end = 0; + size_t name_length = 0; + size_t uncompressed_size = 0; +}; + /// Main compression part int doCompress(char * input, char * output, off_t & in_offset, off_t & out_offset, off_t input_size, off_t output_size, ZSTD_CCtx * cctx) @@ -23,41 +65,27 @@ int doCompress(char * input, char * output, off_t & in_offset, off_t & out_offse } /// compress data from opened file into output file -int compress(int in_fd, int out_fd, int level=5) +int compress(int in_fd, int out_fd, int level, off_t& pointer, const struct stat& info_in) { - /// read data about input file - struct stat info_in; - fstat(in_fd, &info_in); - if (info_in.st_size == 0) { - std::cout << "Empty input file" << std::endl; - return 1; - } - std::cout << "In current size is " << info_in.st_size << std::endl; - - /// Read data about output file. - /// Compressed data will be added to the end of file - /// It will allow to create self extracting executable from file - struct stat info_out; - fstat(out_fd, &info_out); - std::cout << "Out current size is " << info_out.st_size << std::endl; - - /// NOTE: next parametrs depend on binary size - // 6402520 is size of stripped decompressor - size_t start = 6405000ull; - - // 22558008ull size of decompressor - // size_t start = 22558008ull; - /// As experiments showed, size of compressed file is 4 times less than clickhouse executable /// Get a little bit more memory to prevent errors with size. /// For compression this difference will not be huge - ftruncate(out_fd, start + info_in.st_size / 3); - off_t in_offset = 0, out_offset = start; + if (0 != ftruncate(out_fd, pointer + info_in.st_size / 3)) + { + perror(nullptr); + return 1; + } + off_t in_offset = 0; /// mmap files - char * input = static_cast(mmap(nullptr, info_in.st_size, PROT_READ, MAP_PRIVATE , in_fd, 0)); - char * output = static_cast(mmap(nullptr, start + info_in.st_size / 3, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_SHARED, out_fd, 0)); - if (input == reinterpret_cast(-1) || output == reinterpret_cast(-1)) + char * input = static_cast(mmap(nullptr, info_in.st_size, PROT_READ, MAP_PRIVATE, in_fd, 0)); + char * output = static_cast( + mmap(nullptr, pointer + info_in.st_size / 3, + PROT_READ | PROT_WRITE, MAP_SHARED, + out_fd, + 0) + ); + if (input == MAP_FAILED || output == MAP_FAILED) { perror(nullptr); return 1; @@ -65,21 +93,36 @@ int compress(int in_fd, int out_fd, int level=5) /// Create context ZSTD_CCtx * cctx = ZSTD_createCCtx(); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); + if (cctx == nullptr) + { + std::cout << "Failed to create context for compression" << std::endl; + return 1; + } - /// Remember size of file. It will help to avoid using additional memory - /// during decompression - char * file_size = reinterpret_cast(&info_in.st_size); - for (size_t i = 0; i < sizeof(info_in.st_size)/sizeof(char); ++i) - output[out_offset++] = *(file_size + i); + size_t check_result; + /// Set level and enable checksums + check_result = ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level); + if (ZSTD_isError(check_result)) + { + std::cout << "Failed to set compression level: " + std::string(ZSTD_getErrorName(check_result)) << std::endl; + return 1; + } + check_result = ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); + if (ZSTD_isError(check_result)) + { + std::cout << "Failed to set checksums: " + std::string(ZSTD_getErrorName(check_result)) << std::endl; + return 1; + } /// limits for size of block to prevent high memory usage or bad compression off_t max_block_size = 1ull<<27; off_t min_block_size = 1ull<<23; off_t size = 0; - std::cout << in_offset << " " << out_offset << std::endl; + + /// TODO: Maybe make better information instead of offsets + std::cout << "Current offset in infile is|\t Output pointer is " << std::endl; + std::cout << in_offset << "\t\t\t\t" << pointer << std::endl; /// Compress data while (in_offset < info_in.st_size) @@ -92,20 +135,152 @@ int compress(int in_fd, int out_fd, int level=5) size = max_block_size; /// Compress data or exit if error happens - if (0 != doCompress(input, output, in_offset, out_offset, size, ZSTD_compressBound(size), cctx)) + if (0 != doCompress(input, output, in_offset, pointer, size, ZSTD_compressBound(size), cctx)) { - ftruncate(out_fd, info_out.st_size); - munmap(input, info_in.st_size); - munmap(output, start + info_in.st_size / 3); + if (0 != munmap(input, info_in.st_size)) + perror(nullptr); + if (0 != munmap(output, pointer + info_in.st_size / 3)) + perror(nullptr); return 1; } - std::cout << in_offset << " " << out_offset << std::endl; + std::cout << in_offset << "\t\t\t" << pointer << std::endl; } /// Shrink file size and unmap - ftruncate(out_fd, out_offset); - munmap(input, info_in.st_size); - munmap(output, start + info_in.st_size / 3); + if (0 != ftruncate(out_fd, pointer) || 0 != munmap(input, info_in.st_size) || + 0 != munmap(output, pointer + info_in.st_size / 3)) + { + perror(nullptr); + return 1; + } + return 0; +} + +/// Save Metadata at the end of file +int saveMetaData(char* filenames[], int count, int output_fd, const MetaData& metadata, + FileData* files_data, size_t pointer, size_t sum_file_size) +{ + /// Allocate memory for metadata + if (0 != ftruncate(output_fd, pointer + count * sizeof(FileData) + sum_file_size + sizeof(MetaData))) + { + perror(nullptr); + return 1; + } + + char * output = static_cast( + mmap(nullptr, + pointer + count * sizeof(FileData) + sum_file_size + sizeof(MetaData), + PROT_READ | PROT_WRITE, MAP_SHARED, + output_fd, + 0) + ); + if (output == MAP_FAILED) + { + perror(nullptr); + return 1; + } + + /// save information about files and their names + for (int i = 0; i < count; ++i) + { + /// Save file data + memcpy(output + pointer, reinterpret_cast(files_data + i), sizeof(FileData)); + pointer += sizeof(FileData); + + /// Save file name + memcpy(output + pointer, filenames[i], files_data[i].name_length); + pointer += files_data[i].name_length; + } + + /// Save metadata + memcpy(output + pointer, reinterpret_cast(&metadata), sizeof(MetaData)); + return 0; +} + +/// Fills metadata and calls compression function for each file +int compressFiles(char* filenames[], int count, int output_fd, int level, const struct stat& info_out) +{ + /// TODO: check that compression can be done (?) + /// It is difficult to predict compressed size and + /// the upper estimate of memory (size + 1/3 sum_of_files_size) + /// is very rude and can fail even if compression can be successfully done + + MetaData metadata; + size_t sum_file_size = 0; + metadata.number_of_files = count; + off_t pointer = info_out.st_size; + + /// Store information about each file and compress it + FileData* files_data = new FileData[count]; + char * names[count]; + for (int i = 0; i < count; ++i) + { + std::cout << "Start compression for " << filenames[i] << std::endl; + + int input_fd = open(filenames[i], O_RDONLY); + if (input_fd == -1) + { + perror(nullptr); + delete [] files_data; + return 1; + } + + /// Remember information about file name + /// This should be made after the file is opened + /// because filename should be extracted from path + names[i] = strrchr(filenames[i], '/') + 1; + files_data[i].name_length = strlen(names[i]); + sum_file_size += files_data[i].name_length; + + /// read data about input file + struct stat info_in; + if (0 != fstat(input_fd, &info_in)) + { + perror(nullptr); + delete [] files_data; + return 1; + } + + if (info_in.st_size == 0) { + std::cout << "Empty input file will be skipped." << std::endl; + continue; + } + + std::cout << "Input file current size is " << info_in.st_size << std::endl; + + /// Remember information about uncompressed size of file and + /// start of it's compression version + files_data[i].uncompressed_size = info_in.st_size; + files_data[i].start = pointer; + + /// Compressed data will be added to the end of file + /// It will allow to create self extracting executable from file + if (0 != compress(input_fd, output_fd, level, pointer, info_in)) + { + perror(nullptr); + delete [] files_data; + return 1; + } + + /// This error is less important, than others. + /// If file cannot be closed, in some cases it will lead to + /// error in other function that will stop compression process + if (0 != close(input_fd)) + perror(nullptr); + + files_data[i].end = pointer; + } + + /// save location of files information + metadata.start_of_files_data = pointer; + + if (0 != saveMetaData(names, count, output_fd, metadata, files_data, pointer, sum_file_size)) + { + delete [] files_data; + return 1; + } + + delete [] files_data; return 0; } @@ -113,36 +288,54 @@ int main(int argc, char* argv[]) { if (argc < 3) { - std::cout << "Not enough arguments.\ncompressor [file that should be compressed] [file name for compressed file] [OPTIONAL level of compression]" << std::endl; - return 0; - } - - int input_fd = open(argv[1], O_RDONLY); - if (input_fd == -1) - { - perror(nullptr); + std::cout << "Not enough arguments.\ncompressor [OPTIONAL --level of compression] [file name for compressed file] [files that should be compressed]" << std::endl; return 0; } - int output_fd = open(argv[2], O_RDWR | O_CREAT, 0775); + int start_of_files = 1; + + /// Set compression level + int level = 5; + if (0 == memcmp(argv[1], "--level=", 8)) + { + level = strtol(argv[argc - 1], nullptr, 10); + ++start_of_files; + } + + int output_fd = open(argv[start_of_files], O_RDWR | O_CREAT, 0775); if (output_fd == -1) { perror(nullptr); - return 0; + return 1; + } + ++start_of_files; + + struct stat info_out; + if (0 != fstat(output_fd, &info_out)) + { + perror(nullptr); + return 1; } - int result; - if (argc == 4) - result = compress(input_fd, output_fd, strtol(argv[3], nullptr, 10)); + if (0 != compressFiles(&argv[start_of_files], argc - start_of_files, output_fd, level, info_out)) + { + std::cout << "Compression was not successful." << std::endl; + + /// Cancel changes. Reset the file to its original state + if (0 != ftruncate(output_fd, info_out.st_size)) + { + perror(nullptr); + } + } else - result = compress(input_fd, output_fd); - - if (result == 0) + { std::cout << "Successfully compressed" << std::endl; - else - std::cout << "An error has occurred" << std::endl; + } - close(input_fd); - close(output_fd); + if (0 != close(output_fd)) + { + perror(nullptr); + return 1; + } return 0; } diff --git a/utils/self-extr-exec/decompressor.cpp b/utils/self-extr-exec/decompressor.cpp index 2c30342f28b..4a102ad526a 100644 --- a/utils/self-extr-exec/decompressor.cpp +++ b/utils/self-extr-exec/decompressor.cpp @@ -1,13 +1,39 @@ #include #include +#include #include #include +#include #include #include #include #include #include +/* +Metadata contains: + 1) number of files to support multiple file compression + 2) start_of_files_data to know start of files metadata + 3) end of binary to know start of compressed data + 4) uncompressed data size +*/ +struct MetaData +{ + size_t number_of_files = 0; + size_t start_of_files_data = 0; +}; + +/// Information about each file for correct extraction. +/// Each file data is followed by name of file +/// with length equals to name_length. +struct FileData +{ + size_t start = 0; + size_t end = 0; + size_t name_length = 0; + size_t uncompressed_size = 0; +}; + /// decompress part int doDecompress(char * input, char * output, off_t & in_offset, off_t & out_offset, off_t input_size, off_t output_size, ZSTD_DCtx* dctx) @@ -23,65 +49,172 @@ int doDecompress(char * input, char * output, off_t & in_offset, off_t & out_off } /// decompress data from in_fd into out_fd -int decompress(int in_fd, int out_fd) +int decompress(char * input, char * output, off_t start, off_t end) +{ + off_t in_pointer = start, out_pointer = 0; + off_t size = 0; + off_t max_block_size = 1ull<<27; + + /// Create context + ZSTD_DCtx * dctx = ZSTD_createDCtx(); + if (dctx == nullptr) + { + printf("Failed to create context for compression"); + return 1; + } + + /// Compress data + while (in_pointer < end) + { + size = ZSTD_findFrameCompressedSize(input + in_pointer, max_block_size); + + /// Compress data or exit if error happens + if (0 != doDecompress(input, output, in_pointer, out_pointer, size, max_block_size, dctx)) + return 1; + } + + return 0; +} + + +/// Read data about files and decomrpess them. +int decompressFiles(int input_fd, char* argv[]) { /// Read data about output file. /// Compressed data will replace data in file struct stat info_in; - fstat(in_fd, &info_in); - - /// NOTE: next parametrs depend on binary size - // 22558008ull for full, 6405000ull for stripped; - off_t in_offset = 6405000ull /*size of decompressor*/, out_offset = 0; - - /// mmap files - char * input = static_cast(mmap(nullptr, info_in.st_size, PROT_READ, MAP_SHARED , in_fd, 0)); - if (input == reinterpret_cast(-1)) + if (0 != fstat(input_fd, &info_in)) { perror(nullptr); return 1; } - /// Create context - ZSTD_DCtx * dctx = ZSTD_createDCtx(); - - /// Read size of file. It will help to avoid using additional memory - /// during decompression. - size_t * file_size = reinterpret_cast(input + in_offset); - in_offset += sizeof(size_t); - - /// Prepare output file - ftruncate(out_fd, *file_size); - char * output = static_cast(mmap(nullptr, *file_size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_SHARED, out_fd, 0)); - if (output == reinterpret_cast(-1)) + /// mmap input file + char * input = static_cast(mmap(nullptr, info_in.st_size, PROT_READ, MAP_PRIVATE , input_fd, 0)); + if (input == MAP_FAILED) { perror(nullptr); return 1; - } + } - off_t size = 0; - off_t max_block_size = 1ull<<27; + /// Read metadata from end of file + MetaData metadata = *reinterpret_cast(input + info_in.st_size - sizeof(MetaData)); - /// Compress data - while (in_offset < info_in.st_size) + /// Prepare to read information about files and decompress them + off_t files_pointer = metadata.start_of_files_data; + size_t decompressed_full_size = 0; + + /// Read files metadata and check if decompression is possible + off_t check_pointer = metadata.start_of_files_data; + for (size_t i = 0; i < metadata.number_of_files; ++i) { - size = ZSTD_findFrameCompressedSize(input + in_offset, max_block_size); + FileData data = *reinterpret_cast(input + check_pointer); + decompressed_full_size += data.uncompressed_size; + check_pointer += sizeof(FileData) + data.name_length; + } - /// Compress data or exit if error happens - if (0 != doDecompress(input, output, in_offset, out_offset, size, max_block_size, dctx)) + /// Check free space + struct statfs fs_info; + if (0 != fstatfs(input_fd, &fs_info)) + { + perror(nullptr); + if (0 != munmap(input, info_in.st_size)) + perror(nullptr); + return 1; + } + if (fs_info.f_blocks * info_in.st_blksize < decompressed_full_size) + { + printf("Not enough space for decompression. Have %lu, need %zu.", + fs_info.f_blocks * info_in.st_blksize, decompressed_full_size); + return 1; + } + + FileData file_info; + /// Decompress files with appropriate file names + for (size_t i = 0; i < metadata.number_of_files; ++i) + { + /// Read information about file + file_info = *reinterpret_cast(input + files_pointer); + files_pointer += sizeof(FileData); + char file_name[file_info.name_length]; + memcpy(file_name, input + files_pointer, file_info.name_length); + files_pointer += file_info.name_length; + + /// Open file for decompressed data + int output_fd; + /// Check that name differs from executable filename + if (0 == memcmp(file_name, strrchr(argv[0], '/') + 1, file_info.name_length)) { - munmap(input, info_in.st_size); - munmap(output, *file_size); + /// Add .decompressed + char new_file_name[file_info.name_length + 13]; + memcpy(new_file_name, file_name, file_info.name_length); + memcpy(new_file_name + file_info.name_length, ".decompressed", 13); + output_fd = open(new_file_name, O_RDWR | O_CREAT, 0775); + } + else + { + output_fd = open(file_name, O_RDWR | O_CREAT, 0775); + } + if (output_fd == -1) + { + perror(nullptr); + if (0 != munmap(input, info_in.st_size)) + perror(nullptr); + return 0; + } + + /// Prepare output file + if (0 != ftruncate(output_fd, file_info.uncompressed_size)) + { + perror(nullptr); + if (0 != munmap(input, info_in.st_size)) + perror(nullptr); return 1; } - } - /// Shrink file size and unmap - munmap(output, *file_size); - munmap(input, info_in.st_size); + char * output = static_cast( + mmap(nullptr, + file_info.uncompressed_size, + PROT_READ | PROT_WRITE, MAP_SHARED, + output_fd, + 0) + ); + if (output == MAP_FAILED) + { + perror(nullptr); + if (0 != munmap(input, info_in.st_size)) + perror(nullptr); + return 1; + } + + /// Decompress data into file + if (0 != decompress(input, output, file_info.start, file_info.end)) + { + if (0 != munmap(input, info_in.st_size)) + perror(nullptr); + if (0 != munmap(output, file_info.uncompressed_size)) + perror(nullptr); + return 1; + } + + /// TODO: return 1? + if (0 != fsync(output_fd)) + perror(nullptr); + if (0 != close(output_fd)) + perror(nullptr); + } + + if (0 != munmap(input, info_in.st_size)) + perror(nullptr); return 0; } +void fill(char * dest, char * source, size_t length, size_t& shift) +{ + memcpy(dest + shift, source, length); + shift += length; +} + int main(int /*argc*/, char* argv[]) { int input_fd = open(argv[0], O_RDONLY); @@ -90,25 +223,64 @@ int main(int /*argc*/, char* argv[]) perror(nullptr); return 0; } - - int output_fd = open("clickhouse.decompressed", O_RDWR | O_CREAT, 0775); - if (output_fd == -1) - { - perror(nullptr); - return 0; - } - if (0 != decompress(input_fd, output_fd)) + /// Decompress all files + if (0 != decompressFiles(input_fd, argv)) { + printf("Error happend"); + if (0 != close(input_fd)) + perror(nullptr); return 1; } - fsync(output_fd); - close(output_fd); - close(input_fd); + if (0 != close(input_fd)) + perror(nullptr); - /// NOTE: This command should not depend from any variables. - /// It should be changed if file changes. + /// According to documentation `mv` will rename file if it + /// doesn't move to other directory. + /// Sometimes `rename` doesn't exist by default and + /// `rename.ul` is set instead. It will lead to errors + /// that can be easily avoided with help of `mv` + + // /// TODO: decompressor name can differ from executable + // char bash[] = "/usr/bin/bash"; + // size_t length = 0; + // for (int i = 1; i < argc; ++i) + // length += strlen(argv[i]); + // /// mv filename.decompressed filename && filename agrs... + // char command[8 + 3 + strlen(argv[0]) + 14 + strlen(argv[0]) + 4 + strlen(argv[0]) + length + argc - 1]; + // memset(command, '\0', 8 + 3 + strlen(argv[0]) + 14 + strlen(argv[0]) + 4 + strlen(argv[0]) + length + argc - 1); + + // /// fill command + // size_t shift = 0; + // char executable[] = "bash -c "; + // char mv[] = "mv "; + // char decompressed[] = ".decompressed "; + // char add_command[] = " && "; + // char space[] = " "; + // fill(command, executable, 8, shift); + // fill(command, mv, 3, shift); + // fill(command, argv[0], strlen(argv[0]), shift); + // fill(command, decompressed, 14, shift); + // fill(command, argv[0], strlen(argv[0]), shift); + // fill(command, add_command, 4, shift); + // fill(command, argv[0], strlen(argv[0]), shift); + // fill(command, space, 1, shift); + // for (int i = 1; i < argc; ++i) + // { + // fill(command, argv[i], strlen(argv[i]), shift); + // if (i != argc - 1) + // fill(command, space, 1, shift); + // } + // printf("%s", command); + // fflush(stdout); + + // char *newargv[] = { bash, command, nullptr }; + // char *newenviron[] = { nullptr }; + // execve("/usr/bin/bash", newargv, newenviron); + + /// This part of code will be reached only if error happened execl("/usr/bin/bash", "bash", "-c", "mv ./clickhouse.decompressed ./clickhouse", NULL); - return 0; + perror(nullptr); + return 1; } From c01da8b7535d591f35a97ae2f99e3e7ac8c5f16d Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Thu, 7 Apr 2022 12:31:41 +0000 Subject: [PATCH 0033/1954] remove whitespace --- utils/self-extr-exec/decompressor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/self-extr-exec/decompressor.cpp b/utils/self-extr-exec/decompressor.cpp index 4a102ad526a..34c6e0daad5 100644 --- a/utils/self-extr-exec/decompressor.cpp +++ b/utils/self-extr-exec/decompressor.cpp @@ -90,7 +90,7 @@ int decompressFiles(int input_fd, char* argv[]) } /// mmap input file - char * input = static_cast(mmap(nullptr, info_in.st_size, PROT_READ, MAP_PRIVATE , input_fd, 0)); + char * input = static_cast(mmap(nullptr, info_in.st_size, PROT_READ, MAP_PRIVATE, input_fd, 0)); if (input == MAP_FAILED) { perror(nullptr); From 99b14084f0ae47033b6ec50b7d5b88e200722593 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 7 Apr 2022 14:38:56 +0200 Subject: [PATCH 0034/1954] Update contrib/sysroot --- contrib/sysroot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/sysroot b/contrib/sysroot index bbcac834526..e9fb375d0a1 160000 --- a/contrib/sysroot +++ b/contrib/sysroot @@ -1 +1 @@ -Subproject commit bbcac834526d90d1e764164b861be426891d1743 +Subproject commit e9fb375d0a1e5ebfd74c043f088f2342552103f8 From 5b69caf38385061d4c5f8f38cb8c26dff2130e6b Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 7 Apr 2022 15:18:26 +0200 Subject: [PATCH 0035/1954] Update codebrowser clang to 14 --- docker/test/codebrowser/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile index 102c2d4c697..8df03dcf62b 100644 --- a/docker/test/codebrowser/Dockerfile +++ b/docker/test/codebrowser/Dockerfile @@ -15,7 +15,7 @@ RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-14 lib # https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b RUN git clone https://github.com/ClickHouse-Extras/woboq_codebrowser -RUN cd woboq_codebrowser && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-13 -DCMAKE_C_COMPILER=clang-13 && make -j +RUN cd woboq_codebrowser && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-14 -DCMAKE_C_COMPILER=clang-14 && make -j ENV CODEGEN=/woboq_codebrowser/generator/codebrowser_generator ENV CODEINDEX=/woboq_codebrowser/indexgenerator/codebrowser_indexgenerator @@ -28,7 +28,7 @@ ENV SHA=nosha ENV DATA="https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data" CMD mkdir -p $BUILD_DIRECTORY && cd $BUILD_DIRECTORY && \ - cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-13 -DCMAKE_C_COMPILER=/usr/bin/clang-13 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \ + cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-14 -DCMAKE_C_COMPILER=/usr/bin/clang-14 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \ mkdir -p $HTML_RESULT_DIRECTORY && \ $CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA -d $DATA | ts '%Y-%m-%d %H:%M:%S' && \ cp -r $STATIC_DATA $HTML_RESULT_DIRECTORY/ &&\ From bc3e1ec3f290cc4cdd0285fc2f88fc6747756eba Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 7 Apr 2022 17:44:49 +0000 Subject: [PATCH 0036/1954] Part 3 --- .../MergeTree/DataPartStorageOnDisk.h | 4 +- src/Storages/MergeTree/IDataPartStorage.h | 2 + src/Storages/MergeTree/IMergeTreeDataPart.cpp | 48 ++----------- src/Storages/MergeTree/IMergeTreeDataPart.h | 6 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 68 ++++++++++++------- .../MergeTree/MergeTreeDataPartCompact.h | 6 +- .../MergeTree/MergeTreeDataPartInMemory.cpp | 10 ++- .../MergeTree/MergeTreeDataPartInMemory.h | 6 +- .../MergeTree/MergeTreeDataPartWide.cpp | 63 ++++++++++------- .../MergeTree/MergeTreeDataPartWide.h | 6 +- .../MergeTree/MergeTreeIndexGranularityInfo.h | 1 + .../PartMetadataManagerWithCache.cpp | 43 +++++++++--- .../MergeTree/PartMetadataManagerWithCache.h | 2 +- 13 files changed, 137 insertions(+), 128 deletions(-) diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDisk.h index 69858440fe1..e08ef2095f2 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.h +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.h @@ -34,10 +34,12 @@ public: UInt64 calculateTotalSizeOnDisk() const override; + bool isStoredOnRemoteDisk() const override; + void writeChecksums(MergeTreeDataPartChecksums & checksums) const override; void writeColumns(NamesAndTypesList & columns) const override; - bool shallParticipateInMerges(const IStoragePolicy &) const; + bool shallParticipateInMerges(const IStoragePolicy &) const override; void rename(const String & new_relative_path, Poco::Logger * log, bool remove_new_dir_if_exists, bool fsync) override; diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 5af628971bf..9fcdf4a58ca 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -45,6 +45,8 @@ public: virtual UInt64 calculateTotalSizeOnDisk() const = 0; + virtual bool isStoredOnRemoteDisk() const { return false; } + /// Should remove it later virtual void writeChecksums(MergeTreeDataPartChecksums & checksums) const = 0; virtual void writeColumns(NamesAndTypesList & columns) const = 0; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index fc66f547565..bb3ef0c0747 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -299,14 +299,12 @@ IMergeTreeDataPart::IMergeTreeDataPart( const MergeTreeData & storage_, const String & name_, const DataPartStoragePtr & data_part_storage_, - const std::optional & relative_path_, Type part_type_, const IMergeTreeDataPart * parent_part_) : storage(storage_) , name(name_) , info(MergeTreePartInfo::fromPartName(name_, storage.format_version)) , data_part_storage(parent_part_ ? parent_part_->data_part_storage : data_part_storage_) - , relative_path(relative_path_.value_or(name_)) , index_granularity_info(storage_, part_type_) , part_type(part_type_) , parent_part(parent_part_) @@ -327,14 +325,12 @@ IMergeTreeDataPart::IMergeTreeDataPart( const String & name_, const MergeTreePartInfo & info_, const DataPartStoragePtr & data_part_storage_, - const std::optional & relative_path_, Type part_type_, const IMergeTreeDataPart * parent_part_) : storage(storage_) , name(name_) , info(info_) , data_part_storage(parent_part_ ? parent_part_->data_part_storage : data_part_storage_) - , relative_path(relative_path_.value_or(name_)) , index_granularity_info(storage_, part_type_) , part_type(part_type_) , parent_part(parent_part_) @@ -688,7 +684,7 @@ void IMergeTreeDataPart::appendFilesOfColumnsChecksumsIndexes(Strings & files, b Strings projection_files; projection_part->appendFilesOfColumnsChecksumsIndexes(projection_files, true); for (const auto & projection_file : projection_files) - files.push_back(fs::path(projection_part->relative_path) / projection_file); + files.push_back(fs::path(projection_part->name + ".proj") / projection_file); } } } @@ -1222,12 +1218,7 @@ void IMergeTreeDataPart::appendFilesOfColumns(Strings & files) bool IMergeTreeDataPart::shallParticipateInMerges(const StoragePolicyPtr & storage_policy) const { - /// `IMergeTreeDataPart::volume` describes space where current part belongs, and holds - /// `SingleDiskVolume` object which does not contain up-to-date settings of corresponding volume. - /// Therefore we shall obtain volume from storage policy. - auto volume_ptr = storage_policy->getVolume(storage_policy->getVolumeIndexByDisk(volume->getDisk())); - - return !volume_ptr->areMergesAvoided(); + return data_part_storage->shallParticipateInMerges(*storage_policy); } // UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const DataPartStoragePtr & data_part_storage_, const String & from) @@ -1254,43 +1245,12 @@ try "Move is not supported for projection parts: moving form {} to {}", data_part_storage->getFullPath(), new_relative_path); + String from = data_part_storage->getFullRelativePath(); String to = fs::path(storage.relative_data_path) / new_relative_path / ""; - - data_part_storage->move(to); - - if (!volume->getDisk()->exists(from)) - throw Exception("Part directory " + fullPath(volume->getDisk(), from) + " doesn't exist. Most likely it is a logical error.", ErrorCodes::FILE_DOESNT_EXIST); - - if (volume->getDisk()->exists(to)) - { - if (remove_new_dir_if_exists) - { - Names files; - volume->getDisk()->listFiles(to, files); - - LOG_WARNING(storage.log, "Part directory {} already exists and contains {} files. Removing it.", fullPath(volume->getDisk(), to), files.size()); - - volume->getDisk()->removeRecursive(to); - } - else - { - throw Exception("Part directory " + fullPath(volume->getDisk(), to) + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); - } - } - - metadata_manager->deleteAll(true); - metadata_manager->assertAllDeleted(true); - volume->getDisk()->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr))); - volume->getDisk()->moveDirectory(from, to); - relative_path = new_relative_path; - metadata_manager->updateAll(true); + data_part_storage->rename(to, storage.log, remove_new_dir_if_exists, storage.getSettings()->fsync_part_directory); metadata_manager->move(from, to); - SyncGuardPtr sync_guard; - if (storage.getSettings()->fsync_part_directory) - sync_guard = volume->getDisk()->getDirectorySyncGuard(to); - storage.lockSharedData(*this); } catch (...) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index b483d694220..115a6fc65d5 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -70,7 +70,6 @@ public: const String & name_, const MergeTreePartInfo & info_, const DataPartStoragePtr & data_part_storage_, - const std::optional & relative_path, Type part_type_, const IMergeTreeDataPart * parent_part_); @@ -78,7 +77,6 @@ public: const MergeTreeData & storage_, const String & name_, const DataPartStoragePtr & data_part_storage_, - const std::optional & relative_path, Type part_type_, const IMergeTreeDataPart * parent_part_); @@ -200,7 +198,7 @@ public: /// A directory path (relative to storage's path) where part data is actually stored /// Examples: 'detached/tmp_fetch_', 'tmp_', '' /// NOTE: Cannot have trailing slash. - mutable String relative_path; + /// mutable String relative_path; MergeTreeIndexGranularityInfo index_granularity_info; size_t rows_count = 0; @@ -372,7 +370,7 @@ public: bool shallParticipateInMerges(const StoragePolicyPtr & storage_policy) const; /// Calculate the total size of the entire directory with all the files - static UInt64 calculateTotalSizeOnDisk(const DiskPtr & disk_, const String & from); + // static UInt64 calculateTotalSizeOnDisk(const DiskPtr & disk_, const String & from); /// Calculate column and secondary indices sizes on disk. void calculateColumnsAndSecondaryIndicesSizesOnDisk(); diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index f4da730b1f0..00b5ee2e483 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -19,10 +19,9 @@ namespace ErrorCodes MergeTreeDataPartCompact::MergeTreeDataPartCompact( MergeTreeData & storage_, const String & name_, - const VolumePtr & volume_, - const std::optional & relative_path_, + const DataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_) - : IMergeTreeDataPart(storage_, name_, volume_, relative_path_, Type::COMPACT, parent_part_) + : IMergeTreeDataPart(storage_, name_, data_part_storage_, Type::COMPACT, parent_part_) { } @@ -30,10 +29,9 @@ MergeTreeDataPartCompact::MergeTreeDataPartCompact( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const VolumePtr & volume_, - const std::optional & relative_path_, + const DataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_) - : IMergeTreeDataPart(storage_, name_, info_, volume_, relative_path_, Type::COMPACT, parent_part_) + : IMergeTreeDataPart(storage_, name_, info_, data_part_storage_, Type::COMPACT, parent_part_) { } @@ -93,7 +91,7 @@ void MergeTreeDataPartCompact::calculateEachColumnSizes(ColumnSizeByName & /*eac void MergeTreeDataPartCompact::loadIndexGranularity() { - String full_path = getFullRelativePath(); + //String full_path = getFullRelativePath(); if (columns.empty()) throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); @@ -101,13 +99,16 @@ void MergeTreeDataPartCompact::loadIndexGranularity() if (!index_granularity_info.is_adaptive) throw Exception("MergeTreeDataPartCompact cannot be created with non-adaptive granulary.", ErrorCodes::NOT_IMPLEMENTED); - auto marks_file_path = index_granularity_info.getMarksFilePath(full_path + "data"); - if (!volume->getDisk()->exists(marks_file_path)) - throw Exception("Marks file '" + fullPath(volume->getDisk(), marks_file_path) + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); + auto marks_file_path = index_granularity_info.getMarksFilePath("data"); + if (!data_part_storage->exists(marks_file_path)) + throw Exception( + ErrorCodes::NO_FILE_IN_DATA_PART, + "Marks file '{}' doesn't exist", + std::string(fs::path(data_part_storage->getFullPath()) / marks_file_path)); - size_t marks_file_size = volume->getDisk()->getFileSize(marks_file_path); + size_t marks_file_size = data_part_storage->getFileSize(marks_file_path); - auto buffer = volume->getDisk()->readFile(marks_file_path, ReadSettings().adjustBufferSize(marks_file_size), marks_file_size); + auto buffer = data_part_storage->readFile(marks_file_path, ReadSettings().adjustBufferSize(marks_file_size), marks_file_size, std::nullopt); while (!buffer->eof()) { /// Skip offsets for columns @@ -137,7 +138,7 @@ bool MergeTreeDataPartCompact::hasColumnFiles(const NameAndTypePair & column) co void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) const { checkConsistencyBase(); - String path = getFullRelativePath(); + // String path = getFullRelativePath(); String mrk_file_name = DATA_FILE_NAME + index_granularity_info.marks_file_extension; if (!checksums.empty()) @@ -149,42 +150,57 @@ void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) cons if (require_part_metadata) { if (!checksums.files.count(mrk_file_name)) - throw Exception("No marks file checksum for column in part " + fullPath(volume->getDisk(), path), ErrorCodes::NO_FILE_IN_DATA_PART); + throw Exception( + ErrorCodes::NO_FILE_IN_DATA_PART, + "No marks file checksum for column in part {}", + data_part_storage->getFullPath()); if (!checksums.files.count(DATA_FILE_NAME_WITH_EXTENSION)) - throw Exception("No data file checksum for in part " + fullPath(volume->getDisk(), path), ErrorCodes::NO_FILE_IN_DATA_PART); + throw Exception( + ErrorCodes::NO_FILE_IN_DATA_PART, + "No data file checksum for in part {}", + data_part_storage->getFullPath()); } } else { { /// count.txt should be present even in non custom-partitioned parts - auto file_path = path + "count.txt"; - if (!volume->getDisk()->exists(file_path) || volume->getDisk()->getFileSize(file_path) == 0) - throw Exception("Part " + path + " is broken: " + fullPath(volume->getDisk(), file_path) + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + std::string file_path = "count.txt"; + if (!data_part_storage->exists(file_path) || data_part_storage->getFileSize(file_path) == 0) + throw Exception( + ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, + "Part {} is broken: {} is empty", + data_part_storage->getFullRelativePath(), + std::string(fs::path(data_part_storage->getFullPath()) / file_path)); } /// Check that marks are nonempty and have the consistent size with columns number. - auto mrk_file_path = path + mrk_file_name; - if (volume->getDisk()->exists(mrk_file_name)) + if (data_part_storage->exists(mrk_file_name)) { - UInt64 file_size = volume->getDisk()->getFileSize(mrk_file_name); + UInt64 file_size = data_part_storage->getFileSize(mrk_file_name); if (!file_size) - throw Exception("Part " + path + " is broken: " + fullPath(volume->getDisk(), mrk_file_name) + " is empty.", - ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + throw Exception( + ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, + "Part {} is broken: {} is empty.", + data_part_storage->getFullRelativePath(), + std::string(fs::path(data_part_storage->getFullPath()) / mrk_file_name)); UInt64 expected_file_size = index_granularity_info.getMarkSizeInBytes(columns.size()) * index_granularity.getMarksCount(); if (expected_file_size != file_size) throw Exception( - "Part " + path + " is broken: bad size of marks file '" + fullPath(volume->getDisk(), mrk_file_name) + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), - ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, + "Part {} is broken: bad size of marks file '{}': {}, must be: {}", + data_part_storage->getFullRelativePath(), + std::string(fs::path(data_part_storage->getFullPath()) / mrk_file_name), + std::to_string(file_size), std::to_string(expected_file_size)); } } } bool MergeTreeDataPartCompact::isStoredOnRemoteDisk() const { - return volume->getDisk()->isRemote(); + return data_part_storage->isStoredOnRemoteDisk(); } MergeTreeDataPartCompact::~MergeTreeDataPartCompact() diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 38bfa11652a..48d5ef3cb73 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -25,15 +25,13 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const VolumePtr & volume_, - const std::optional & relative_path_ = {}, + const DataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_ = nullptr); MergeTreeDataPartCompact( MergeTreeData & storage_, const String & name_, - const VolumePtr & volume_, - const std::optional & relative_path_ = {}, + const DataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_ = nullptr); MergeTreeReaderPtr getReader( diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp index e4a174a7d29..2925f244d59 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp @@ -20,10 +20,9 @@ namespace ErrorCodes MergeTreeDataPartInMemory::MergeTreeDataPartInMemory( MergeTreeData & storage_, const String & name_, - const VolumePtr & volume_, - const std::optional & relative_path_, + const DataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_) - : IMergeTreeDataPart(storage_, name_, volume_, relative_path_, Type::IN_MEMORY, parent_part_) + : IMergeTreeDataPart(storage_, name_, data_part_storage_, Type::IN_MEMORY, parent_part_) { default_codec = CompressionCodecFactory::instance().get("NONE", {}); } @@ -32,10 +31,9 @@ MergeTreeDataPartInMemory::MergeTreeDataPartInMemory( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const VolumePtr & volume_, - const std::optional & relative_path_, + const DataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_) - : IMergeTreeDataPart(storage_, name_, info_, volume_, relative_path_, Type::IN_MEMORY, parent_part_) + : IMergeTreeDataPart(storage_, name_, info_, data_part_storage_, Type::IN_MEMORY, parent_part_) { default_codec = CompressionCodecFactory::instance().get("NONE", {}); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.h b/src/Storages/MergeTree/MergeTreeDataPartInMemory.h index c5ee9ebd01f..d015bb01b64 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.h +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.h @@ -14,15 +14,13 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const VolumePtr & volume_, - const std::optional & relative_path_ = {}, + const DataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_ = nullptr); MergeTreeDataPartInMemory( MergeTreeData & storage_, const String & name_, - const VolumePtr & volume_, - const std::optional & relative_path_ = {}, + const DataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_ = nullptr); MergeTreeReaderPtr getReader( diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index b279c1aba6a..eec863510eb 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -21,10 +21,9 @@ namespace ErrorCodes MergeTreeDataPartWide::MergeTreeDataPartWide( MergeTreeData & storage_, const String & name_, - const VolumePtr & volume_, - const std::optional & relative_path_, + const DataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_) - : IMergeTreeDataPart(storage_, name_, volume_, relative_path_, Type::WIDE, parent_part_) + : IMergeTreeDataPart(storage_, name_, data_part_storage_, Type::WIDE, parent_part_) { } @@ -32,10 +31,9 @@ MergeTreeDataPartWide::MergeTreeDataPartWide( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const VolumePtr & volume_, - const std::optional & relative_path_, + const DataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_) - : IMergeTreeDataPart(storage_, name_, info_, volume_, relative_path_, Type::WIDE, parent_part_) + : IMergeTreeDataPart(storage_, name_, info_, data_part_storage_, Type::WIDE, parent_part_) { } @@ -104,19 +102,20 @@ ColumnSize MergeTreeDataPartWide::getColumnSizeImpl( void MergeTreeDataPartWide::loadIndexGranularity() { - String full_path = getFullRelativePath(); - index_granularity_info.changeGranularityIfRequired(volume->getDisk(), full_path); + index_granularity_info.changeGranularityIfRequired(data_part_storage); if (columns.empty()) throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); /// We can use any column, it doesn't matter - std::string marks_file_path = index_granularity_info.getMarksFilePath(full_path + getFileNameForColumn(columns.front())); - if (!volume->getDisk()->exists(marks_file_path)) - throw Exception("Marks file '" + fullPath(volume->getDisk(), marks_file_path) + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); + std::string marks_file_path = index_granularity_info.getMarksFilePath(getFileNameForColumn(columns.front())); + if (!data_part_storage->exists(marks_file_path)) + throw Exception( + ErrorCodes::NO_FILE_IN_DATA_PART, "Marks file '{}' doesn't exist", + std::string(fs::path(data_part_storage->getFullPath()) / marks_file_path)); - size_t marks_file_size = volume->getDisk()->getFileSize(marks_file_path); + size_t marks_file_size = data_part_storage->getFileSize(marks_file_path); if (!index_granularity_info.is_adaptive) { @@ -125,7 +124,7 @@ void MergeTreeDataPartWide::loadIndexGranularity() } else { - auto buffer = volume->getDisk()->readFile(marks_file_path, ReadSettings().adjustBufferSize(marks_file_size), marks_file_size); + auto buffer = data_part_storage->readFile(marks_file_path, ReadSettings().adjustBufferSize(marks_file_size), marks_file_size, std::nullopt); while (!buffer->eof()) { buffer->seek(sizeof(size_t) * 2, SEEK_CUR); /// skip offset_in_compressed file and offset_in_decompressed_block @@ -135,7 +134,9 @@ void MergeTreeDataPartWide::loadIndexGranularity() } if (index_granularity.getMarksCount() * index_granularity_info.getMarkSizeInBytes() != marks_file_size) - throw Exception("Cannot read all marks from file " + fullPath(volume->getDisk(), marks_file_path), ErrorCodes::CANNOT_READ_ALL_DATA); + throw Exception( + ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read all marks from file {}", + std::string(fs::path(data_part_storage->getFullPath()) / marks_file_path)); } index_granularity.setInitialized(); @@ -154,7 +155,7 @@ MergeTreeDataPartWide::~MergeTreeDataPartWide() void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const { checkConsistencyBase(); - String path = getFullRelativePath(); + //String path = getFullRelativePath(); if (!checksums.empty()) { @@ -167,12 +168,18 @@ void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const String file_name = ISerialization::getFileNameForStream(name_type, substream_path); String mrk_file_name = file_name + index_granularity_info.marks_file_extension; String bin_file_name = file_name + DATA_FILE_EXTENSION; + if (!checksums.files.count(mrk_file_name)) - throw Exception("No " + mrk_file_name + " file checksum for column " + name_type.name + " in part " + fullPath(volume->getDisk(), path), - ErrorCodes::NO_FILE_IN_DATA_PART); + throw Exception( + ErrorCodes::NO_FILE_IN_DATA_PART, + "No {} file checksum for column {} in part {} ", + mrk_file_name, name_type.name, data_part_storage->getFullPath()); + if (!checksums.files.count(bin_file_name)) - throw Exception("No " + bin_file_name + " file checksum for column " + name_type.name + " in part " + fullPath(volume->getDisk(), path), - ErrorCodes::NO_FILE_IN_DATA_PART); + throw Exception( + ErrorCodes::NO_FILE_IN_DATA_PART, + "No {} file checksum for column {} in part ", + bin_file_name, name_type.name, data_part_storage->getFullPath()); }); } } @@ -185,22 +192,26 @@ void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const { getSerialization(name_type)->enumerateStreams([&](const ISerialization::SubstreamPath & substream_path) { - auto file_path = path + ISerialization::getFileNameForStream(name_type, substream_path) + index_granularity_info.marks_file_extension; + auto file_path = ISerialization::getFileNameForStream(name_type, substream_path) + index_granularity_info.marks_file_extension; /// Missing file is Ok for case when new column was added. - if (volume->getDisk()->exists(file_path)) + if (data_part_storage->exists(file_path)) { - UInt64 file_size = volume->getDisk()->getFileSize(file_path); + UInt64 file_size = data_part_storage->getFileSize(file_path); if (!file_size) - throw Exception("Part " + path + " is broken: " + fullPath(volume->getDisk(), file_path) + " is empty.", - ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + throw Exception( + ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, + "Part {} is broken: {} is empty.", + data_part_storage->getFullPath(), + std::string(fs::path(data_part_storage->getFullPath()) / file_path)); if (!marks_size) marks_size = file_size; else if (file_size != *marks_size) - throw Exception("Part " + path + " is broken: marks have different sizes.", - ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + throw Exception( + ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, + "Part {} is broken: marks have different sizes.", data_part_storage->getFullPath()); } }); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.h b/src/Storages/MergeTree/MergeTreeDataPartWide.h index 078dda394ee..81f9b65a5bc 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -19,15 +19,13 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const VolumePtr & volume, - const std::optional & relative_path_ = {}, + const DataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_ = nullptr); MergeTreeDataPartWide( MergeTreeData & storage_, const String & name_, - const VolumePtr & volume, - const std::optional & relative_path_ = {}, + const DataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_ = nullptr); MergeTreeReaderPtr getReader( diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index 1893c25e22c..dbb027c244e 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp b/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp index 3d68497f5b0..e6516afa222 100644 --- a/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp +++ b/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp @@ -40,7 +40,7 @@ String PartMetadataManagerWithCache::getFilePathFromKey(const String & key) cons std::unique_ptr PartMetadataManagerWithCache::read(const String & file_name) const { - String file_path = fs::path(part->getFullRelativePath()) / file_name; + String file_path = fs::path(part->data_part_storage->getFullRelativePath()) / file_name; String key = getKeyFromFilePath(file_path); String value; auto status = cache->get(key, value); @@ -60,7 +60,12 @@ std::unique_ptr PartMetadataManagerWithCache::read(const Str bool PartMetadataManagerWithCache::exists(const String & file_name) const { - String file_path = fs::path(part->getFullRelativePath()) / file_name; + return existsImpl(part->data_part_storage->getFullRelativePath(), file_name); +} + +bool PartMetadataManagerWithCache::existsImpl(const String & path, const String & file_name) const +{ + String file_path = fs::path(path) / file_name; String key = getKeyFromFilePath(file_path); String value; auto status = cache->get(key, value); @@ -72,11 +77,16 @@ bool PartMetadataManagerWithCache::exists(const String & file_name) const else { ProfileEvents::increment(ProfileEvents::MergeTreeMetadataCacheMiss); - return disk->exists(fs::path(part->getFullRelativePath()) / file_name); + return disk->exists(fs::path(path) / file_name); } } void PartMetadataManagerWithCache::deleteAll(bool include_projection) +{ + deleteAllImpl(part->data_part_storage->getFullRelativePath(), include_projection); +} + +void PartMetadataManagerWithCache::deleteAllImpl(const String & path, bool include_projection) { Strings file_names; part->appendFilesOfColumnsChecksumsIndexes(file_names, include_projection); @@ -84,7 +94,7 @@ void PartMetadataManagerWithCache::deleteAll(bool include_projection) String value; for (const auto & file_name : file_names) { - String file_path = fs::path(part->getFullRelativePath()) / file_name; + String file_path = fs::path(path) / file_name; String key = getKeyFromFilePath(file_path); auto status = cache->del(key); if (!status.ok()) @@ -104,6 +114,11 @@ void PartMetadataManagerWithCache::deleteAll(bool include_projection) } void PartMetadataManagerWithCache::updateAll(bool include_projection) +{ + updateAllImpl(part->data_part_storage->getFullRelativePath(), include_projection); +} + +void PartMetadataManagerWithCache::updateAllImpl(const String & path, bool include_projection) { Strings file_names; part->appendFilesOfColumnsChecksumsIndexes(file_names, include_projection); @@ -112,7 +127,7 @@ void PartMetadataManagerWithCache::updateAll(bool include_projection) String read_value; for (const auto & file_name : file_names) { - String file_path = fs::path(part->getFullRelativePath()) / file_name; + String file_path = fs::path(path) / file_name; if (!disk->exists(file_path)) continue; auto in = disk->readFile(file_path); @@ -137,6 +152,11 @@ void PartMetadataManagerWithCache::updateAll(bool include_projection) } void PartMetadataManagerWithCache::assertAllDeleted(bool include_projection) const +{ + assertAllDeletedImpl(part->data_part_storage->getFullRelativePath(), include_projection); +} + +void PartMetadataManagerWithCache::assertAllDeletedImpl(const String & path, bool include_projection) const { Strings keys; std::vector _; @@ -152,7 +172,7 @@ void PartMetadataManagerWithCache::assertAllDeleted(bool include_projection) con file_name = fs::path(file_path).filename(); /// Metadata file belongs to current part - if (fs::path(part->getFullRelativePath()) / file_name == file_path) + if (fs::path(path) / file_name == file_path) throw Exception( ErrorCodes::LOGICAL_ERROR, "Data part {} with type {} with meta file {} still in cache", @@ -166,7 +186,7 @@ void PartMetadataManagerWithCache::assertAllDeleted(bool include_projection) con const auto & projection_parts = part->getProjectionParts(); for (const auto & [projection_name, projection_part] : projection_parts) { - if (fs::path(projection_part->getFullRelativePath()) / file_name == file_path) + if (fs::path(path) / (projection_name + ".proj") / file_name == file_path) { throw Exception( ErrorCodes::LOGICAL_ERROR, @@ -181,9 +201,16 @@ void PartMetadataManagerWithCache::assertAllDeleted(bool include_projection) con } } +void PartMetadataManagerWithCache::move(const String & from, const String & to) +{ + deleteAllImpl(from, true); + assertAllDeletedImpl(from, true); + updateAllImpl(to, true); +} + void PartMetadataManagerWithCache::getKeysAndCheckSums(Strings & keys, std::vector & checksums) const { - String prefix = getKeyFromFilePath(fs::path(part->getFullRelativePath()) / ""); + String prefix = getKeyFromFilePath(fs::path(part->data_part_storage->getFullRelativePath()) / ""); Strings values; cache->getByPrefix(prefix, keys, values); size_t size = keys.size(); diff --git a/src/Storages/MergeTree/PartMetadataManagerWithCache.h b/src/Storages/MergeTree/PartMetadataManagerWithCache.h index 856ea991611..676b931be9c 100644 --- a/src/Storages/MergeTree/PartMetadataManagerWithCache.h +++ b/src/Storages/MergeTree/PartMetadataManagerWithCache.h @@ -50,7 +50,7 @@ private: /// Get cache keys and checksums of corresponding metadata in a part(including projection parts) void getKeysAndCheckSums(Strings & keys, std::vector & checksums) const; - + bool existsImpl(const String & path, const String & file_name) const; void deleteAllImpl(const String & path, bool include_projection); void assertAllDeletedImpl(const String & path, bool include_projection) const; void updateAllImpl(const String & path, bool include_projection); From 20aee231bd736a333ad04c773985a6df9ceafaeb Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 8 Apr 2022 13:41:39 +0200 Subject: [PATCH 0037/1954] Improve codebrowser image, use a temporary branch as source --- docker/test/codebrowser/Dockerfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile index 8df03dcf62b..c7aed618f6a 100644 --- a/docker/test/codebrowser/Dockerfile +++ b/docker/test/codebrowser/Dockerfile @@ -13,9 +13,13 @@ RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-14 lib # repo versions doesn't work correctly with C++17 # also we push reports to s3, so we add index.html to subfolder urls # https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b -RUN git clone https://github.com/ClickHouse-Extras/woboq_codebrowser - -RUN cd woboq_codebrowser && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-14 -DCMAKE_C_COMPILER=clang-14 && make -j +# TODO: remove branch in a few weeks after merge, e.g. in May or June 2022 +RUN git clone https://github.com/ClickHouse-Extras/woboq_codebrowser --branch llvm-14 \ + && cd woboq_codebrowser \ + && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-14 -DCMAKE_C_COMPILER=clang-14 \ + && make -j \ + && cd .. \ + && rm -rf woboq_codebrowser ENV CODEGEN=/woboq_codebrowser/generator/codebrowser_generator ENV CODEINDEX=/woboq_codebrowser/indexgenerator/codebrowser_indexgenerator From 7c4b28863891c52f53cdae441c1978dec42a38a9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 8 Apr 2022 18:56:08 +0000 Subject: [PATCH 0038/1954] Part 4 --- .../MergeTree/DataPartStorageOnDisk.cpp | 29 ++++++++ .../MergeTree/DataPartStorageOnDisk.h | 4 ++ src/Storages/MergeTree/IDataPartStorage.h | 4 ++ src/Storages/MergeTree/MergeTreeData.cpp | 69 +++++++++---------- .../MergeTree/MergeTreeDataWriter.cpp | 9 ++- 5 files changed, 75 insertions(+), 40 deletions(-) diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 0ec9acaa0c6..2ead1129978 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -43,6 +43,11 @@ bool DataPartStorageOnDisk::exists() const return volume->getDisk()->exists(root_path); } +Poco::Timestamp DataPartStorageOnDisk::getLastModified() const +{ + return volume->getDisk()->getLastModified(root_path); +} + size_t DataPartStorageOnDisk::getFileSize(const String & path) const { return volume->getDisk()->getFileSize(fs::path(root_path) / path); @@ -80,6 +85,16 @@ UInt64 DataPartStorageOnDisk::calculateTotalSizeOnDisk() const return calculateTotalSizeOnDiskImpl(volume->getDisk(), root_path); } +bool DataPartStorageOnDisk::isStoredOnRemoteDisk() const +{ + return volume->getDisk()->isRemote(); +} + +bool DataPartStorageOnDisk::supportZeroCopyReplication() const +{ + return volume->getDisk()->supportZeroCopyReplication(); +} + void DataPartStorageOnDisk::writeChecksums(MergeTreeDataPartChecksums & checksums) const { std::string path = fs::path(root_path) / "checksums.txt"; @@ -104,6 +119,20 @@ void DataPartStorageOnDisk::writeColumns(NamesAndTypesList & columns) const volume->getDisk()->moveFile(path + ".tmp", path); } +void DataPartStorageOnDisk::writeDeleteOnDestroyMarker(Poco::Logger * log) const +{ + String marker_path = fs::path(root_path) / "delete-on-destroy.txt"; + auto disk = volume->getDisk(); + try + { + volume->getDisk()->createFile(marker_path); + } + catch (Poco::Exception & e) + { + LOG_ERROR(log, "{} (while creating DeleteOnDestroy marker: {})", e.what(), backQuote(fullPath(disk, marker_path))); + } +} + void DataPartStorageOnDisk::rename(const String & new_relative_path, Poco::Logger * log, bool remove_new_dir_if_exists, bool fsync) { if (!volume->getDisk()->exists(root_path)) diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDisk.h index e08ef2095f2..a45c11529a8 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.h +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.h @@ -24,6 +24,8 @@ public: bool exists(const std::string & path) const override; bool exists() const override; + Poco::Timestamp getLastModified() const override; + size_t getFileSize(const std::string & path) const override; DiskDirectoryIteratorPtr iterate() const override; @@ -35,9 +37,11 @@ public: UInt64 calculateTotalSizeOnDisk() const override; bool isStoredOnRemoteDisk() const override; + bool supportZeroCopyReplication() const override; void writeChecksums(MergeTreeDataPartChecksums & checksums) const override; void writeColumns(NamesAndTypesList & columns) const override; + void writeDeleteOnDestroyMarker(Poco::Logger * log) const override; bool shallParticipateInMerges(const IStoragePolicy &) const override; diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 9fcdf4a58ca..038e5d6f9dc 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -35,6 +35,8 @@ public: virtual bool exists(const std::string & path) const = 0; virtual bool exists() const = 0; + virtual Poco::Timestamp getLastModified() const = 0; + virtual DiskDirectoryIteratorPtr iterate() const = 0; virtual DiskDirectoryIteratorPtr iterateDirectory(const std::string & path) const = 0; @@ -46,10 +48,12 @@ public: virtual UInt64 calculateTotalSizeOnDisk() const = 0; virtual bool isStoredOnRemoteDisk() const { return false; } + virtual bool supportZeroCopyReplication() const { return false; } /// Should remove it later virtual void writeChecksums(MergeTreeDataPartChecksums & checksums) const = 0; virtual void writeColumns(NamesAndTypesList & columns) const = 0; + virtual void writeDeleteOnDestroyMarker(Poco::Logger * log) const = 0; /// A leak of abstraction virtual bool shallParticipateInMerges(const IStoragePolicy &) const { return true; } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 6c4ef0ce6fa..38639328c07 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -1008,7 +1009,8 @@ void MergeTreeData::loadDataPartsFromDisk( return; const auto & part_info = *part_opt; auto single_disk_volume = std::make_shared("volume_" + part_name, part_disk_ptr, 0); - auto part = createPart(part_name, part_info, single_disk_volume, part_name); + auto data_part_storage = std::make_shared(single_disk_volume, relative_data_path, part_name); + auto part = createPart(part_name, part_info, data_part_storage); bool broken = false; String part_path = fs::path(relative_data_path) / part_name; @@ -1016,7 +1018,7 @@ void MergeTreeData::loadDataPartsFromDisk( if (part_disk_ptr->exists(marker_path)) { /// NOTE: getBytesOnDisk() cannot be used here, since it maybe zero of checksums.txt will not exist - size_t size_of_part = IMergeTreeDataPart::calculateTotalSizeOnDisk(part->volume->getDisk(), part->getFullRelativePath()); + size_t size_of_part = data_part_storage->calculateTotalSizeOnDisk(); LOG_WARNING(log, "Detaching stale part {}{} (size: {}), which should have been deleted after a move. " "That can only happen after unclean restart of ClickHouse after move of a part having an operation blocking that stale copy of part.", @@ -1053,7 +1055,7 @@ void MergeTreeData::loadDataPartsFromDisk( if (broken) { /// NOTE: getBytesOnDisk() cannot be used here, since it maybe zero of checksums.txt will not exist - size_t size_of_part = IMergeTreeDataPart::calculateTotalSizeOnDisk(part->volume->getDisk(), part->getFullRelativePath()); + size_t size_of_part = data_part_storage->calculateTotalSizeOnDisk(); LOG_ERROR(log, "Detaching broken part {}{} (size: {}). " @@ -1082,7 +1084,7 @@ void MergeTreeData::loadDataPartsFromDisk( { if ((*it)->checksums.getTotalChecksumHex() == part->checksums.getTotalChecksumHex()) { - LOG_ERROR(log, "Remove duplicate part {}", part->getFullPath()); + LOG_ERROR(log, "Remove duplicate part {}", data_part_storage->getFullPath()); duplicate_parts_to_remove.push_back(part); } else @@ -1182,7 +1184,7 @@ void MergeTreeData::loadDataPartsFromWAL( { if ((*it)->checksums.getTotalChecksumHex() == part->checksums.getTotalChecksumHex()) { - LOG_ERROR(log, "Remove duplicate part {}", part->getFullPath()); + LOG_ERROR(log, "Remove duplicate part {}", part->data_part_storage->getFullPath()); duplicate_parts_to_remove.push_back(part); } else @@ -1584,7 +1586,7 @@ void MergeTreeData::flushAllInMemoryPartsIfNeeded() if (auto part_in_memory = asInMemoryPart(part)) { const auto & storage_relative_path = part_in_memory->storage.relative_data_path; - part_in_memory->flushToDisk(storage_relative_path, part_in_memory->relative_path, metadata_snapshot); + part_in_memory->flushToDisk(storage_relative_path, part_in_memory->data_part_storage->getRelativePath(), metadata_snapshot); } } @@ -2273,7 +2275,7 @@ MergeTreeDataPartType MergeTreeData::choosePartTypeOnDisk(size_t bytes_uncompres MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, MergeTreeDataPartType type, const MergeTreePartInfo & part_info, - const DataPartStoragePtr & data_part_storage, const String & relative_path, const IMergeTreeDataPart * parent_part) const + const DataPartStoragePtr & data_part_storage, const IMergeTreeDataPart * parent_part) const { if (type == MergeTreeDataPartType::COMPACT) return std::make_shared(*this, name, part_info, data_part_storage, parent_part); @@ -2282,7 +2284,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, else if (type == MergeTreeDataPartType::IN_MEMORY) return std::make_shared(*this, name, part_info, data_part_storage, parent_part); else - throw Exception("Unknown type of part " + relative_path, ErrorCodes::UNKNOWN_PART_TYPE); + throw Exception("Unknown type of part " + data_part_storage->getFullRelativePath(), ErrorCodes::UNKNOWN_PART_TYPE); } static MergeTreeDataPartType getPartTypeFromMarkExtension(const String & mrk_ext) @@ -2298,14 +2300,14 @@ static MergeTreeDataPartType getPartTypeFromMarkExtension(const String & mrk_ext } MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( - const String & name, const DataPartStoragePtr & data_part_storage, const String & relative_path, const IMergeTreeDataPart * parent_part) const + const String & name, const DataPartStoragePtr & data_part_storage, const IMergeTreeDataPart * parent_part) const { - return createPart(name, MergeTreePartInfo::fromPartName(name, format_version), data_part_storage, relative_path, parent_part); + return createPart(name, MergeTreePartInfo::fromPartName(name, format_version), data_part_storage, parent_part); } MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( const String & name, const MergeTreePartInfo & part_info, - const DataPartStoragePtr & data_part_storage, const String & relative_path, const IMergeTreeDataPart * parent_part) const + const DataPartStoragePtr & data_part_storage, const IMergeTreeDataPart * parent_part) const { MergeTreeDataPartType type; // auto full_path = fs::path(relative_data_path) / (parent_part ? parent_part->relative_path : "") / relative_path / ""; @@ -2319,7 +2321,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( type = choosePartTypeOnDisk(0, 0); } - return createPart(name, type, part_info, data_part_storage, relative_path, parent_part); + return createPart(name, type, part_info, data_part_storage, parent_part); } void MergeTreeData::changeSettings( @@ -2564,7 +2566,7 @@ bool MergeTreeData::renameTempPartAndReplace( else /// Parts from ReplicatedMergeTree already have names part_name = part->name; - LOG_TRACE(log, "Renaming temporary part {} to {}.", part->relative_path, part_name); + LOG_TRACE(log, "Renaming temporary part {} to {}.", part->data_part_storage->getRelativePath(), part_name); if (auto it_duplicate = data_parts_by_info.find(part_info); it_duplicate != data_parts_by_info.end()) { @@ -2827,9 +2829,9 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c void MergeTreeData::forgetPartAndMoveToDetached(const MergeTreeData::DataPartPtr & part_to_detach, const String & prefix, bool restore_covered) { if (prefix.empty()) - LOG_INFO(log, "Renaming {} to {} and forgetting it.", part_to_detach->relative_path, part_to_detach->name); + LOG_INFO(log, "Renaming {} to {} and forgetting it.", part_to_detach->data_part_storage->getRelativePath(), part_to_detach->name); else - LOG_INFO(log, "Renaming {} to {}_{} and forgetting it.", part_to_detach->relative_path, prefix, part_to_detach->name); + LOG_INFO(log, "Renaming {} to {}_{} and forgetting it.", part_to_detach->data_part_storage->getRelativePath(), prefix, part_to_detach->name); auto lock = lockParts(); bool removed_active_part = false; @@ -3191,8 +3193,8 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy) original_active_part->force_keep_shared_data = false; - if (original_active_part->volume->getDisk()->supportZeroCopyReplication() && - part_copy->volume->getDisk()->supportZeroCopyReplication() && + if (original_active_part->data_part_storage->supportZeroCopyReplication() && + part_copy->data_part_storage->supportZeroCopyReplication() && original_active_part->getUniqueId() == part_copy->getUniqueId()) { /// May be when several volumes use the same S3/HDFS storage @@ -3209,16 +3211,7 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy) ssize_t diff_rows = part_copy->rows_count - original_active_part->rows_count; increaseDataVolume(diff_bytes, diff_rows, /* parts= */ 0); - auto disk = original_active_part->volume->getDisk(); - String marker_path = fs::path(original_active_part->getFullRelativePath()) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; - try - { - disk->createFile(marker_path); - } - catch (Poco::Exception & e) - { - LOG_ERROR(log, "{} (while creating DeleteOnDestroy marker: {})", e.what(), backQuote(fullPath(disk, marker_path))); - } + original_active_part->data_part_storage->writeDeleteOnDestroyMarker(log); return; } } @@ -3287,11 +3280,8 @@ MergeTreeData::DataPartPtr MergeTreeData::getPartIfExists(const String & part_na static void loadPartAndFixMetadataImpl(MergeTreeData::MutableDataPartPtr part) { - auto disk = part->volume->getDisk(); - String full_part_path = part->getFullRelativePath(); - part->loadColumnsChecksumsIndexes(false, true); - part->modification_time = disk->getLastModified(full_part_path).epochTime(); + part->modification_time = part->data_part_storage->getLastModified().epochTime(); } void MergeTreeData::calculateColumnAndSecondaryIndexSizesImpl() @@ -3443,7 +3433,7 @@ void MergeTreeData::movePartitionToDisk(const ASTPtr & partition, const String & auto disk = getStoragePolicy()->getDiskByName(name); parts.erase(std::remove_if(parts.begin(), parts.end(), [&](auto part_ptr) { - return part_ptr->volume->getDisk()->getName() == disk->getName(); + return part_ptr->data_part_storage->getName() == disk->getName(); }), parts.end()); if (parts.empty()) @@ -3493,7 +3483,7 @@ void MergeTreeData::movePartitionToVolume(const ASTPtr & partition, const String { for (const auto & disk : volume->getDisks()) { - if (part_ptr->volume->getDisk()->getName() == disk->getName()) + if (part_ptr->data_part_storage->getName() == disk->getName()) { return true; } @@ -3798,7 +3788,8 @@ private: } auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); - auto part = storage->createPart(part_name, part_info, single_disk_volume, relative_temp_part_dir); + auto data_part_storage = std::make_shared(single_disk_volume, relative_data_path, relative_temp_part_dir); + auto part = storage->createPart(part_name, part_info, data_part_storage); part->loadColumnsChecksumsIndexes(false, true); storage->renameTempPartAndAdd(part, increment); return {}; @@ -4173,7 +4164,8 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const LOG_DEBUG(log, "Checking part {}", new_name); auto single_disk_volume = std::make_shared("volume_" + old_name, disk); - MutableDataPartPtr part = createPart(old_name, single_disk_volume, source_dir + new_name); + auto data_part_storage = std::make_shared(single_disk_volume, relative_data_path, source_dir + new_name); + MutableDataPartPtr part = createPart(old_name, data_part_storage); loadPartAndFixMetadataImpl(part); loaded_parts.push_back(part); @@ -4392,7 +4384,7 @@ void MergeTreeData::Transaction::rollbackPartsToTemporaryState() WriteBufferFromOwnString buf; buf << " Rollbacking parts state to temporary and removing from working set:"; for (const auto & part : precommitted_parts) - buf << " " << part->relative_path; + buf << " " << part->data_part_storage->getRelativePath(); buf << "."; LOG_DEBUG(data.log, "Undoing transaction.{}", buf.str()); @@ -4410,7 +4402,7 @@ void MergeTreeData::Transaction::rollback() WriteBufferFromOwnString buf; buf << " Removing parts:"; for (const auto & part : precommitted_parts) - buf << " " << part->relative_path; + buf << " " << part->data_part_storage->getRelativePath(); buf << "."; LOG_DEBUG(data.log, "Undoing transaction.{}", buf.str()); @@ -5361,7 +5353,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk( disk->removeFileIfExists(fs::path(dst_part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); - auto dst_data_part = createPart(dst_part_name, dst_part_info, single_disk_volume, tmp_dst_part_name); + auto data_part_storage = std::make_shared(single_disk_volume, relative_data_path, tmp_dst_part_name); + auto dst_data_part = createPart(dst_part_name, dst_part_info, data_part_storage); dst_data_part->is_temp = true; diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index da43f5c2cb0..a206d7a5f55 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -367,12 +368,16 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPart( ReservationPtr reservation = data.reserveSpacePreferringTTLRules(metadata_snapshot, expected_size, move_ttl_infos, time(nullptr), 0, true); VolumePtr volume = data.getStoragePolicy()->getVolume(0); + auto data_part_storage = std::make_shared( + createVolumeFromReservation(reservation, volume), + data.relative_data_path, + TMP_PREFIX + part_name); + auto new_data_part = data.createPart( part_name, data.choosePartType(expected_size, block.rows()), new_part_info, - createVolumeFromReservation(reservation, volume), - TMP_PREFIX + part_name); + data_part_storage); if (data.storage_settings.get()->assign_part_uuids) new_data_part->uuid = UUIDHelpers::generateV4(); From 8b90bd06a566a7cf846f9e26f593ed87e6811bf7 Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Sat, 9 Apr 2022 15:16:58 -0400 Subject: [PATCH 0039/1954] Improve decompression in readbig for avoiding unnecessary memcpy --- .../CompressedReadBufferFromFile.cpp | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/Compression/CompressedReadBufferFromFile.cpp b/src/Compression/CompressedReadBufferFromFile.cpp index 9efb3c92cde..333734af3d6 100644 --- a/src/Compression/CompressedReadBufferFromFile.cpp +++ b/src/Compression/CompressedReadBufferFromFile.cpp @@ -117,6 +117,25 @@ size_t CompressedReadBufferFromFile::readBig(char * to, size_t n) bytes_read += size_decompressed; bytes += size_decompressed; } + else if (nextimpl_working_buffer_offset > 0) + { + //Need to skip some bytes in decompressed data (seek happened before readBig call). + size_compressed = new_size_compressed; + bytes += offset(); + + /// This is for clang static analyzer. + assert(size_decompressed + additional_size_at_the_end_of_buffer > 0); + memory.resize(size_decompressed + additional_size_at_the_end_of_buffer); + working_buffer = Buffer(memory.data(), &memory[size_decompressed]); + decompress(working_buffer, size_decompressed, size_compressed_without_checksum); + + /// Read partial data from first block. We don't use nextImpl in this method + /// Avoid to call unnecessary memcpy in read when second block fits entirely to output buffer + size_t size_partial = std::min((size_decompressed - nextimpl_working_buffer_offset),(n - bytes_read)); + pos = working_buffer.begin() + nextimpl_working_buffer_offset; + nextimpl_working_buffer_offset = 0; + bytes_read += read(to + bytes_read, size_partial); + } else { size_compressed = new_size_compressed; @@ -124,17 +143,12 @@ size_t CompressedReadBufferFromFile::readBig(char * to, size_t n) /// This is for clang static analyzer. assert(size_decompressed + additional_size_at_the_end_of_buffer > 0); - memory.resize(size_decompressed + additional_size_at_the_end_of_buffer); working_buffer = Buffer(memory.data(), &memory[size_decompressed]); - decompress(working_buffer, size_decompressed, size_compressed_without_checksum); - /// Manually take nextimpl_working_buffer_offset into account, because we don't use - /// nextImpl in this method. - pos = working_buffer.begin() + nextimpl_working_buffer_offset; - nextimpl_working_buffer_offset = 0; - + ///Read partial data from last block. We don't use nextImpl in this method as well. + pos = working_buffer.begin(); bytes_read += read(to + bytes_read, n - bytes_read); break; } From 547aeca83faf38e555f12f8d5b70b79672fa6331 Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Sat, 9 Apr 2022 16:14:38 -0400 Subject: [PATCH 0040/1954] fixed comments --- src/Compression/CompressedReadBufferFromFile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compression/CompressedReadBufferFromFile.cpp b/src/Compression/CompressedReadBufferFromFile.cpp index 333734af3d6..a876c77f1e8 100644 --- a/src/Compression/CompressedReadBufferFromFile.cpp +++ b/src/Compression/CompressedReadBufferFromFile.cpp @@ -119,7 +119,7 @@ size_t CompressedReadBufferFromFile::readBig(char * to, size_t n) } else if (nextimpl_working_buffer_offset > 0) { - //Need to skip some bytes in decompressed data (seek happened before readBig call). + /// Need to skip some bytes in decompressed data (seek happened before readBig call). size_compressed = new_size_compressed; bytes += offset(); From c99fef5d82cc10e72790a9fa70a3f73e48b086c1 Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Sat, 9 Apr 2022 16:31:24 -0400 Subject: [PATCH 0041/1954] Modifiy comments --- src/Compression/CompressedReadBufferFromFile.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Compression/CompressedReadBufferFromFile.cpp b/src/Compression/CompressedReadBufferFromFile.cpp index a876c77f1e8..9c7cd4b273a 100644 --- a/src/Compression/CompressedReadBufferFromFile.cpp +++ b/src/Compression/CompressedReadBufferFromFile.cpp @@ -129,8 +129,8 @@ size_t CompressedReadBufferFromFile::readBig(char * to, size_t n) working_buffer = Buffer(memory.data(), &memory[size_decompressed]); decompress(working_buffer, size_decompressed, size_compressed_without_checksum); - /// Read partial data from first block. We don't use nextImpl in this method - /// Avoid to call unnecessary memcpy in read when second block fits entirely to output buffer + /// Read partial data from first block. + /// Avoid to call unnecessary nextImpl+memcpy in read when second block fits entirely to output buffer size_t size_partial = std::min((size_decompressed - nextimpl_working_buffer_offset),(n - bytes_read)); pos = working_buffer.begin() + nextimpl_working_buffer_offset; nextimpl_working_buffer_offset = 0; @@ -147,7 +147,7 @@ size_t CompressedReadBufferFromFile::readBig(char * to, size_t n) working_buffer = Buffer(memory.data(), &memory[size_decompressed]); decompress(working_buffer, size_decompressed, size_compressed_without_checksum); - ///Read partial data from last block. We don't use nextImpl in this method as well. + ///Read partial data from last block. pos = working_buffer.begin(); bytes_read += read(to + bytes_read, n - bytes_read); break; From 0875fd087fc9f60c5d83d7eea2f3322b732b7ead Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Sat, 9 Apr 2022 17:53:08 -0400 Subject: [PATCH 0042/1954] Modify comments --- src/Compression/CompressedReadBufferFromFile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compression/CompressedReadBufferFromFile.cpp b/src/Compression/CompressedReadBufferFromFile.cpp index 9c7cd4b273a..8845c9816a4 100644 --- a/src/Compression/CompressedReadBufferFromFile.cpp +++ b/src/Compression/CompressedReadBufferFromFile.cpp @@ -130,7 +130,7 @@ size_t CompressedReadBufferFromFile::readBig(char * to, size_t n) decompress(working_buffer, size_decompressed, size_compressed_without_checksum); /// Read partial data from first block. - /// Avoid to call unnecessary nextImpl+memcpy in read when second block fits entirely to output buffer + /// Avoid to call nextImpl and unnecessary memcpy in read when the second block fits entirely to output buffer size_t size_partial = std::min((size_decompressed - nextimpl_working_buffer_offset),(n - bytes_read)); pos = working_buffer.begin() + nextimpl_working_buffer_offset; nextimpl_working_buffer_offset = 0; From a54c01cf723d993cb24df60cf076d74fd8973fcf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 11 Apr 2022 00:44:30 +0200 Subject: [PATCH 0043/1954] Remove useless code in ReplicatedMergeTreeRestartingThread --- .../Impl/ParallelFormattingOutputFormat.cpp | 13 +++++++------ .../MergeTree/MergeFromLogEntryTask.cpp | 1 - .../ReplicatedMergeTreeRestartingThread.cpp | 19 ------------------- src/Storages/StorageReplicatedMergeTree.cpp | 3 +-- 4 files changed, 8 insertions(+), 28 deletions(-) diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp index 26be258a34f..dd43cb1f7f7 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp @@ -178,40 +178,41 @@ namespace DB switch (unit.type) { - case ProcessingUnitType::START : + case ProcessingUnitType::START: { formatter->writePrefix(); break; } - case ProcessingUnitType::PLAIN : + case ProcessingUnitType::PLAIN: { formatter->consume(std::move(unit.chunk)); break; } - case ProcessingUnitType::PLAIN_FINISH : + case ProcessingUnitType::PLAIN_FINISH: { formatter->writeSuffix(); break; } - case ProcessingUnitType::TOTALS : + case ProcessingUnitType::TOTALS: { formatter->consumeTotals(std::move(unit.chunk)); break; } - case ProcessingUnitType::EXTREMES : + case ProcessingUnitType::EXTREMES: { if (are_totals_written) formatter->setTotalsAreWritten(); formatter->consumeExtremes(std::move(unit.chunk)); break; } - case ProcessingUnitType::FINALIZE : + case ProcessingUnitType::FINALIZE: { formatter->setOutsideStatistics(std::move(unit.statistics)); formatter->finalizeImpl(); break; } } + /// Flush all the data to handmade buffer. formatter->flush(); unit.actual_memory_size = out_buffer.getActualSize(); diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index 9459849b90a..91be83ace16 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -181,7 +181,6 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare() future_merged_part->updatePath(storage, reserved_space.get()); future_merged_part->merge_type = entry.merge_type; - if (storage_settings_ptr->allow_remote_fs_zero_copy_replication) { if (auto disk = reserved_space->getDisk(); disk->getType() == DB::DiskType::S3) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index dc52660f1f6..9b4234a99e5 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -304,25 +304,6 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() String is_active_path = fs::path(storage.replica_path) / "is_active"; - /** If the node is marked as active, but the mark is made in the same instance, delete it. - * This is possible only when session in ZooKeeper expires. - */ - String data; - Coordination::Stat stat; - bool has_is_active = zookeeper->tryGet(is_active_path, data, &stat); - if (has_is_active && data == active_node_identifier) - { - auto code = zookeeper->tryRemove(is_active_path, stat.version); - - if (code == Coordination::Error::ZBADVERSION) - throw Exception("Another instance of replica " + storage.replica_path + " was created just now." - " You shouldn't run multiple instances of same replica. You need to check configuration files.", - ErrorCodes::REPLICA_IS_ALREADY_ACTIVE); - - if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNONODE) - throw Coordination::Exception(code, is_active_path); - } - /// Simultaneously declare that this replica is active, and update the host. Coordination::Requests ops; ops.emplace_back(zkutil::makeCreateRequest(is_active_path, active_node_identifier, zkutil::CreateMode::Ephemeral)); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 66a5baf555b..9dc733739a5 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -35,7 +35,6 @@ #include #include - #include #include @@ -84,6 +83,7 @@ #include #include + namespace fs = std::filesystem; namespace ProfileEvents @@ -5735,7 +5735,6 @@ void StorageReplicatedMergeTree::fetchPartition( String best_replica; { - /// List of replicas of source shard. replicas = zookeeper->getChildren(fs::path(from) / "replicas"); From d21d6befc1618c84e60255833e2fee6f7fb3d4b8 Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Mon, 11 Apr 2022 05:41:40 -0400 Subject: [PATCH 0044/1954] Modify comments --- src/Compression/CompressedReadBufferFromFile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compression/CompressedReadBufferFromFile.cpp b/src/Compression/CompressedReadBufferFromFile.cpp index 8845c9816a4..05fe7ee56c9 100644 --- a/src/Compression/CompressedReadBufferFromFile.cpp +++ b/src/Compression/CompressedReadBufferFromFile.cpp @@ -129,7 +129,7 @@ size_t CompressedReadBufferFromFile::readBig(char * to, size_t n) working_buffer = Buffer(memory.data(), &memory[size_decompressed]); decompress(working_buffer, size_decompressed, size_compressed_without_checksum); - /// Read partial data from first block. + /// Read partial data from first block. Won't run here at second block /// Avoid to call nextImpl and unnecessary memcpy in read when the second block fits entirely to output buffer size_t size_partial = std::min((size_decompressed - nextimpl_working_buffer_offset),(n - bytes_read)); pos = working_buffer.begin() + nextimpl_working_buffer_offset; From 62341229321b83b1755018b89c03c736a5400cae Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Mon, 11 Apr 2022 06:02:39 -0400 Subject: [PATCH 0045/1954] Modify comment --- src/Compression/CompressedReadBufferFromFile.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Compression/CompressedReadBufferFromFile.cpp b/src/Compression/CompressedReadBufferFromFile.cpp index 05fe7ee56c9..dc8bba511f9 100644 --- a/src/Compression/CompressedReadBufferFromFile.cpp +++ b/src/Compression/CompressedReadBufferFromFile.cpp @@ -129,8 +129,8 @@ size_t CompressedReadBufferFromFile::readBig(char * to, size_t n) working_buffer = Buffer(memory.data(), &memory[size_decompressed]); decompress(working_buffer, size_decompressed, size_compressed_without_checksum); - /// Read partial data from first block. Won't run here at second block - /// Avoid to call nextImpl and unnecessary memcpy in read when the second block fits entirely to output buffer + /// Read partial data from first block. Won't run here at second block. + /// Avoid to call nextImpl and unnecessary memcpy in read when the second block fits entirely to output buffer. size_t size_partial = std::min((size_decompressed - nextimpl_working_buffer_offset),(n - bytes_read)); pos = working_buffer.begin() + nextimpl_working_buffer_offset; nextimpl_working_buffer_offset = 0; From ba112c230275e202c97598e43b1e976c03d3da97 Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Mon, 11 Apr 2022 06:41:55 -0400 Subject: [PATCH 0046/1954] Fixed style issue --- src/Backups/registerBackupEnginesFileAndDisk.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Backups/registerBackupEnginesFileAndDisk.cpp b/src/Backups/registerBackupEnginesFileAndDisk.cpp index e3b06a21d96..fa1786c6350 100644 --- a/src/Backups/registerBackupEnginesFileAndDisk.cpp +++ b/src/Backups/registerBackupEnginesFileAndDisk.cpp @@ -53,7 +53,8 @@ namespace } /// Checks that a path specified as parameters of File() is valid. - void checkPath(fs::path & path, const Poco::Util::AbstractConfiguration & config, const fs::path & data_dir) { + void checkPath(fs::path & path, const Poco::Util::AbstractConfiguration & config, const fs::path & data_dir) + { path = path.lexically_normal(); if (path.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path to backup must not be empty"); From 76870ad92a7fa4e4c301639163cbff9b4dacf7f1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 12 Apr 2022 18:59:49 +0000 Subject: [PATCH 0047/1954] Part 5 --- .../MergeTree/DataPartStorageOnDisk.cpp | 133 ++++++++++++++---- .../MergeTree/DataPartStorageOnDisk.h | 39 ++++- src/Storages/MergeTree/IDataPartStorage.h | 34 ++++- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 17 +-- src/Storages/MergeTree/IMergeTreeDataPart.h | 5 +- .../MergeTree/IMergeTreeDataPartWriter.cpp | 2 + .../MergeTree/IMergeTreeDataPartWriter.h | 2 + src/Storages/MergeTree/IMergeTreeReader.cpp | 6 +- .../MergeTree/IMergedBlockOutputStream.cpp | 5 +- .../MergeTree/IMergedBlockOutputStream.h | 5 +- src/Storages/MergeTree/MergeTreeData.cpp | 6 + src/Storages/MergeTree/MergeTreeData.h | 1 + .../MergeTree/MergeTreeDataPartCompact.cpp | 3 +- .../MergeTree/MergeTreeDataPartCompact.h | 1 + .../MergeTree/MergeTreeDataPartInMemory.cpp | 1 + .../MergeTree/MergeTreeDataPartInMemory.h | 1 + .../MergeTree/MergeTreeDataPartWide.cpp | 5 +- .../MergeTree/MergeTreeDataPartWide.h | 1 + .../MergeTreeDataPartWriterCompact.cpp | 17 ++- .../MergeTreeDataPartWriterCompact.h | 1 + .../MergeTreeDataPartWriterInMemory.cpp | 2 +- .../MergeTreeDataPartWriterOnDisk.cpp | 24 ++-- .../MergeTree/MergeTreeDataPartWriterOnDisk.h | 5 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 22 +-- .../MergeTree/MergeTreeDataPartWriterWide.h | 1 + .../MergeTree/MergeTreeDataWriter.cpp | 56 +++++--- src/Storages/MergeTree/MergeTreeDataWriter.h | 4 + .../MergeTree/MergeTreeMarksLoader.cpp | 16 ++- src/Storages/MergeTree/MergeTreeMarksLoader.h | 6 +- src/Storages/MergeTree/MergeTreePartition.cpp | 8 +- src/Storages/MergeTree/MergeTreePartition.h | 6 +- .../MergeTree/MergeTreeReaderCompact.cpp | 24 ++-- .../MergeTree/MergeTreeReaderStream.cpp | 17 +-- .../MergeTree/MergeTreeReaderStream.h | 4 +- .../MergeTree/MergeTreeReaderWide.cpp | 7 +- src/Storages/MergeTree/MergeTreeReaderWide.h | 1 - .../MergeTree/MergedBlockOutputStream.cpp | 40 +++--- .../MergeTree/MergedBlockOutputStream.h | 1 + 38 files changed, 364 insertions(+), 165 deletions(-) diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 2ead1129978..a33d91715db 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -12,16 +12,22 @@ namespace ErrorCodes { extern const int FILE_DOESNT_EXIST; extern const int DIRECTORY_ALREADY_EXISTS; + extern const int NOT_ENOUGH_SPACE; } -DataPartStorageOnDisk::DataPartStorageOnDisk(VolumePtr volume_, std::string root_path_) - : volume(std::move(volume_)), root_path(std::move(root_path_)) +DataPartStorageOnDisk::DataPartStorageOnDisk(VolumePtr volume_, std::string root_path_, std::string part_dir_) + : volume(std::move(volume_)), root_path(std::move(root_path_)), part_dir(std::move(part_dir_)) { } +std::string DataPartStorageOnDisk::getFullRelativePath() const +{ + return fs::path(root_path) / part_dir; +} + std::string DataPartStorageOnDisk::getFullPath() const { - return fs::path(volume->getDisk()->getPath()) / root_path; + return fs::path(volume->getDisk()->getPath()) / root_path / part_dir; } std::unique_ptr DataPartStorageOnDisk::readFile( @@ -30,42 +36,42 @@ std::unique_ptr DataPartStorageOnDisk::readFile( std::optional read_hint, std::optional file_size) const { - return volume->getDisk()->readFile(fs::path(root_path) / path, settings, read_hint, file_size); + return volume->getDisk()->readFile(fs::path(root_path) / part_dir / path, settings, read_hint, file_size); } bool DataPartStorageOnDisk::exists(const std::string & path) const { - return volume->getDisk()->exists(fs::path(root_path) / path); + return volume->getDisk()->exists(fs::path(root_path) / part_dir / path); } bool DataPartStorageOnDisk::exists() const { - return volume->getDisk()->exists(root_path); + return volume->getDisk()->exists(fs::path(root_path) / part_dir); } Poco::Timestamp DataPartStorageOnDisk::getLastModified() const { - return volume->getDisk()->getLastModified(root_path); + return volume->getDisk()->getLastModified(fs::path(root_path) / part_dir); } size_t DataPartStorageOnDisk::getFileSize(const String & path) const { - return volume->getDisk()->getFileSize(fs::path(root_path) / path); + return volume->getDisk()->getFileSize(fs::path(root_path) / part_dir / path); } DiskDirectoryIteratorPtr DataPartStorageOnDisk::iterate() const { - return volume->getDisk()->iterateDirectory(root_path); + return volume->getDisk()->iterateDirectory(fs::path(root_path) / part_dir); } DiskDirectoryIteratorPtr DataPartStorageOnDisk::iterateDirectory(const String & path) const { - return volume->getDisk()->iterateDirectory(fs::path(root_path) / path); + return volume->getDisk()->iterateDirectory(fs::path(root_path) / part_dir / path); } DataPartStoragePtr DataPartStorageOnDisk::getProjection(const std::string & name) const { - return std::make_shared(volume, fs::path(root_path) / name); + return std::make_shared(volume, std::string(fs::path(root_path) / part_dir), name); } static UInt64 calculateTotalSizeOnDiskImpl(const DiskPtr & disk, const String & from) @@ -82,7 +88,7 @@ static UInt64 calculateTotalSizeOnDiskImpl(const DiskPtr & disk, const String & UInt64 DataPartStorageOnDisk::calculateTotalSizeOnDisk() const { - return calculateTotalSizeOnDiskImpl(volume->getDisk(), root_path); + return calculateTotalSizeOnDiskImpl(volume->getDisk(), fs::path(root_path) / part_dir); } bool DataPartStorageOnDisk::isStoredOnRemoteDisk() const @@ -97,7 +103,7 @@ bool DataPartStorageOnDisk::supportZeroCopyReplication() const void DataPartStorageOnDisk::writeChecksums(MergeTreeDataPartChecksums & checksums) const { - std::string path = fs::path(root_path) / "checksums.txt"; + std::string path = fs::path(root_path) / part_dir / "checksums.txt"; { auto out = volume->getDisk()->writeFile(path + ".tmp", 4096); @@ -109,7 +115,7 @@ void DataPartStorageOnDisk::writeChecksums(MergeTreeDataPartChecksums & checksum void DataPartStorageOnDisk::writeColumns(NamesAndTypesList & columns) const { - std::string path = fs::path(root_path) / "columns.txt"; + std::string path = fs::path(root_path) / part_dir / "columns.txt"; { auto buf = volume->getDisk()->writeFile(path + ".tmp", 4096); @@ -121,7 +127,7 @@ void DataPartStorageOnDisk::writeColumns(NamesAndTypesList & columns) const void DataPartStorageOnDisk::writeDeleteOnDestroyMarker(Poco::Logger * log) const { - String marker_path = fs::path(root_path) / "delete-on-destroy.txt"; + String marker_path = fs::path(root_path) / part_dir / "delete-on-destroy.txt"; auto disk = volume->getDisk(); try { @@ -135,11 +141,14 @@ void DataPartStorageOnDisk::writeDeleteOnDestroyMarker(Poco::Logger * log) const void DataPartStorageOnDisk::rename(const String & new_relative_path, Poco::Logger * log, bool remove_new_dir_if_exists, bool fsync) { - if (!volume->getDisk()->exists(root_path)) - throw Exception("Part directory " + fullPath(volume->getDisk(), root_path) + " doesn't exist. Most likely it is a logical error.", ErrorCodes::FILE_DOESNT_EXIST); + if (!exists()) + throw Exception( + ErrorCodes::FILE_DOESNT_EXIST, + "Part directory {} doesn't exist. Most likely it is a logical error.", + std::string(fs::path(volume->getDisk()->getPath()) / root_path / part_dir)); - /// Why? - String to = fs::path(new_relative_path) / ""; + /// Why "" ? + String to = fs::path(root_path) / new_relative_path / ""; if (volume->getDisk()->exists(to)) { @@ -148,28 +157,35 @@ void DataPartStorageOnDisk::rename(const String & new_relative_path, Poco::Logge Names files; volume->getDisk()->listFiles(to, files); - LOG_WARNING(log, "Part directory {} already exists and contains {} files. Removing it.", fullPath(volume->getDisk(), to), files.size()); + LOG_WARNING(log, + "Part directory {} already exists and contains {} files. Removing it.", + fullPath(volume->getDisk(), to), files.size()); volume->getDisk()->removeRecursive(to); } else { - throw Exception("Part directory " + fullPath(volume->getDisk(), to) + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); + throw Exception( + ErrorCodes::DIRECTORY_ALREADY_EXISTS, + "Part directory {} already exists", + fullPath(volume->getDisk(), to)); } } // metadata_manager->deleteAll(true); // metadata_manager->assertAllDeleted(true); + String from = getFullRelativePath(); + /// Why? - volume->getDisk()->setLastModified(root_path, Poco::Timestamp::fromEpochTime(time(nullptr))); - volume->getDisk()->moveDirectory(root_path, to); - root_path = new_relative_path; + volume->getDisk()->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr))); + volume->getDisk()->moveDirectory(from, to); + part_dir = new_relative_path; // metadata_manager->updateAll(true); SyncGuardPtr sync_guard; if (fsync) - sync_guard = volume->getDisk()->getDirectorySyncGuard(root_path); + sync_guard = volume->getDisk()->getDirectorySyncGuard(getFullRelativePath()); } bool DataPartStorageOnDisk::shallParticipateInMerges(const IStoragePolicy & storage_policy) const @@ -187,4 +203,71 @@ std::string DataPartStorageOnDisk::getName() const return volume->getDisk()->getName(); } + +DataPartStorageBuilderOnDisk::DataPartStorageBuilderOnDisk(VolumePtr volume_, std::string root_path_, std::string part_dir_) + : volume(std::move(volume_)), root_path(std::move(root_path_)), part_dir(std::move(part_dir_)) +{ +} + +std::unique_ptr DataPartStorageBuilderOnDisk::readFile( + const std::string & path, + const ReadSettings & settings, + std::optional read_hint, + std::optional file_size) const +{ + return volume->getDisk()->readFile(fs::path(root_path) / part_dir / path, settings, read_hint, file_size); +} + +std::unique_ptr DataPartStorageBuilderOnDisk::writeFile( + const String & path, + size_t buf_size) +{ + return volume->getDisk()->writeFile(fs::path(root_path) / part_dir / path, buf_size); +} + +void DataPartStorageBuilderOnDisk::removeFile(const String & path) +{ + return volume->getDisk()->removeFile(fs::path(root_path) / part_dir / path); +} + +void DataPartStorageBuilderOnDisk::removeRecursive() +{ + volume->getDisk()->removeRecursive(fs::path(root_path) / part_dir); +} + +bool DataPartStorageBuilderOnDisk::exists() const +{ + return volume->getDisk()->exists(fs::path(root_path) / part_dir); +} + + +bool DataPartStorageBuilderOnDisk::exists(const std::string & path) const +{ + return volume->getDisk()->exists(fs::path(root_path) / part_dir / path); +} + +std::string DataPartStorageBuilderOnDisk::getFullPath() const +{ + return fs::path(volume->getDisk()->getPath()) / root_path / part_dir; +} + +void DataPartStorageBuilderOnDisk::createDirectories() +{ + return volume->getDisk()->createDirectories(fs::path(root_path) / part_dir); +} + +ReservationPtr DataPartStorageBuilderOnDisk::reserve(UInt64 bytes) +{ + auto res = volume->reserve(bytes); + if (!res) + throw Exception(ErrorCodes::NOT_ENOUGH_SPACE, "Cannot reserve {}, not enough space", ReadableSize(bytes)); + + return res; +} + +DataPartStorageBuilderPtr DataPartStorageBuilderOnDisk::getProjection(const std::string & name) const +{ + return std::make_shared(volume, std::string(fs::path(root_path) / part_dir), name); +} + } diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDisk.h index a45c11529a8..118be4dae82 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.h +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.h @@ -13,7 +13,7 @@ using VolumePtr = std::shared_ptr; class DataPartStorageOnDisk final : public IDataPartStorage { public: - explicit DataPartStorageOnDisk(VolumePtr volume_, std::string root_path_); + DataPartStorageOnDisk(VolumePtr volume_, std::string root_path_, std::string part_dir_); std::unique_ptr readFile( const std::string & path, @@ -32,7 +32,7 @@ public: DiskDirectoryIteratorPtr iterateDirectory(const std::string & path) const override; std::string getFullPath() const override; - std::string getFullRelativePath() const override { return root_path; } + std::string getFullRelativePath() const override; UInt64 calculateTotalSizeOnDisk() const override; @@ -54,6 +54,41 @@ public: private: VolumePtr volume; std::string root_path; + std::string part_dir; +}; + +class DataPartStorageBuilderOnDisk final : public IDataPartStorageBuilder +{ + DataPartStorageBuilderOnDisk(VolumePtr volume_, std::string root_path_, std::string part_dir_); + + bool exists() const override; + bool exists(const std::string & path) const override; + + void createDirectories() override; + + std::string getFullPath() const override; + + std::unique_ptr readFile( + const std::string & path, + const ReadSettings & settings, + std::optional read_hint, + std::optional file_size) const override; + + std::unique_ptr writeFile( + const String & path, + size_t buf_size) override; + + void removeFile(const String & path) override; + void removeRecursive() override; + + ReservationPtr reserve(UInt64 bytes) override; + + DataPartStorageBuilderPtr getProjection(const std::string & name) const override; + +private: + VolumePtr volume; + std::string root_path; + std::string part_dir; }; } diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 038e5d6f9dc..640bfdb6b7e 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -8,6 +8,7 @@ namespace DB { class ReadBufferFromFileBase; +class WriteBufferFromFileBase; class IDiskDirectoryIterator; @@ -15,6 +16,9 @@ using DiskDirectoryIteratorPtr = std::unique_ptr; struct MergeTreeDataPartChecksums; +class IReservation; +using ReservationPtr = std::unique_ptr; + class IStoragePolicy; /// This is an abstraction of storage for data part files. @@ -25,7 +29,7 @@ private: public: virtual ~IDataPartStorage() = default; - /// Open the file for read and return ReadBufferFromFileBase object. + /// Open the file for read and return ReadBufferFromFileBase object. virtual std::unique_ptr readFile( const std::string & path, const ReadSettings & settings, @@ -66,12 +70,38 @@ public: virtual std::shared_ptr getProjection(const std::string & name) const = 0; }; +using DataPartStoragePtr = std::shared_ptr; + class IDataPartStorageBuilder { public: virtual ~IDataPartStorageBuilder() = default; + + virtual bool exists() const = 0; + virtual bool exists(const std::string & path) const = 0; + + virtual std::string getFullPath() const = 0; + + virtual void createDirectories() = 0; + + virtual std::unique_ptr readFile( + const std::string & path, + const ReadSettings & settings, + std::optional read_hint, + std::optional file_size) const = 0; + + virtual std::unique_ptr writeFile( + const String & path, + size_t buf_size /* = DBMS_DEFAULT_BUFFER_SIZE*/) = 0; + + virtual void removeFile(const String & path) = 0; + virtual void removeRecursive() = 0; + + virtual ReservationPtr reserve(UInt64 /*bytes*/) { return nullptr; } + + virtual std::shared_ptr getProjection(const std::string & name) const = 0; }; -using DataPartStoragePtr = std::shared_ptr; +using DataPartStorageBuilderPtr = std::shared_ptr; } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index bb3ef0c0747..97bdb59d285 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -98,7 +98,7 @@ void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const Par } IMergeTreeDataPart::MinMaxIndex::WrittenFiles IMergeTreeDataPart::MinMaxIndex::store( - const MergeTreeData & data, const DiskPtr & disk_, const String & part_path, Checksums & out_checksums) const + const MergeTreeData & data, const DataPartStorageBuilderPtr & data_part_storage_builder, Checksums & out_checksums) const { auto metadata_snapshot = data.getInMemoryMetadataPtr(); const auto & partition_key = metadata_snapshot->getPartitionKey(); @@ -106,19 +106,20 @@ IMergeTreeDataPart::MinMaxIndex::WrittenFiles IMergeTreeDataPart::MinMaxIndex::s auto minmax_column_names = data.getMinMaxColumnsNames(partition_key); auto minmax_column_types = data.getMinMaxColumnsTypes(partition_key); - return store(minmax_column_names, minmax_column_types, disk_, part_path, out_checksums); + return store(minmax_column_names, minmax_column_types, data_part_storage_builder, out_checksums); } IMergeTreeDataPart::MinMaxIndex::WrittenFiles IMergeTreeDataPart::MinMaxIndex::store( const Names & column_names, const DataTypes & data_types, - const DiskPtr & disk_, - const String & part_path, + const DataPartStorageBuilderPtr & data_part_storage_builder, Checksums & out_checksums) const { if (!initialized) - throw Exception("Attempt to store uninitialized MinMax index for part " + part_path + ". This is a bug.", - ErrorCodes::LOGICAL_ERROR); + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Attempt to store uninitialized MinMax index for part {}. This is a bug.", + data_part_storage_builder->getFullPath()); WrittenFiles written_files; @@ -127,7 +128,7 @@ IMergeTreeDataPart::MinMaxIndex::WrittenFiles IMergeTreeDataPart::MinMaxIndex::s String file_name = "minmax_" + escapeForFileName(column_names[i]) + ".idx"; auto serialization = data_types.at(i)->getDefaultSerialization(); - auto out = disk_->writeFile(fs::path(part_path) / file_name); + auto out = data_part_storage_builder->writeFile(file_name, DBMS_DEFAULT_BUFFER_SIZE); HashingWriteBuffer out_hashing(*out); serialization->serializeBinary(hyperrectangle[i].left, out_hashing); serialization->serializeBinary(hyperrectangle[i].right, out_hashing); @@ -1248,7 +1249,7 @@ try String from = data_part_storage->getFullRelativePath(); String to = fs::path(storage.relative_data_path) / new_relative_path / ""; - data_part_storage->rename(to, storage.log, remove_new_dir_if_exists, storage.getSettings()->fsync_part_directory); + data_part_storage->rename(new_relative_path, storage.log, remove_new_dir_if_exists, storage.getSettings()->fsync_part_directory); metadata_manager->move(from, to); storage.lockSharedData(*this); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 115a6fc65d5..6c12f2c84f1 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -91,6 +91,7 @@ public: const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = ReadBufferFromFileBase::ProfileCallback{}) const = 0; virtual MergeTreeWriterPtr getWriter( + DataPartStorageBuilderPtr data_part_storage_builder, const NamesAndTypesList & columns_list, const StorageMetadataPtr & metadata_snapshot, const std::vector & indices_to_recalc, @@ -307,8 +308,8 @@ public: using WrittenFiles = std::vector>; - [[nodiscard]] WrittenFiles store(const MergeTreeData & data, const DiskPtr & disk_, const String & part_path, Checksums & checksums) const; - [[nodiscard]] WrittenFiles store(const Names & column_names, const DataTypes & data_types, const DiskPtr & disk_, const String & part_path, Checksums & checksums) const; + [[nodiscard]] WrittenFiles store(const MergeTreeData & data, const DataPartStorageBuilderPtr & data_part_storage_builder, Checksums & checksums) const; + [[nodiscard]] WrittenFiles store(const Names & column_names, const DataTypes & data_types, const DataPartStorageBuilderPtr & data_part_storage_builder, Checksums & checksums) const; void update(const Block & block, const Names & column_names); void merge(const MinMaxIndex & other); diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 3d6b12efdf1..84d0b50ae2f 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -39,11 +39,13 @@ Block permuteBlockIfNeeded(const Block & block, const IColumn::Permutation * per IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( const MergeTreeData::DataPartPtr & data_part_, + DataPartStorageBuilderPtr data_part_storage_builder_, const NamesAndTypesList & columns_list_, const StorageMetadataPtr & metadata_snapshot_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) : data_part(data_part_) + , data_part_storage_builder(std::move(data_part_storage_builder_)) , storage(data_part_->storage) , metadata_snapshot(metadata_snapshot_) , columns_list(columns_list_) diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 34c53eda846..417e2713180 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -23,6 +23,7 @@ class IMergeTreeDataPartWriter : private boost::noncopyable public: IMergeTreeDataPartWriter( const MergeTreeData::DataPartPtr & data_part_, + DataPartStorageBuilderPtr data_part_storage_builder_, const NamesAndTypesList & columns_list_, const StorageMetadataPtr & metadata_snapshot_, const MergeTreeWriterSettings & settings_, @@ -42,6 +43,7 @@ public: protected: const MergeTreeData::DataPartPtr data_part; + DataPartStorageBuilderPtr data_part_storage_builder; const MergeTreeData & storage; const StorageMetadataPtr metadata_snapshot; const NamesAndTypesList columns_list; diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 3a823345dda..b8aeb8e6a5a 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -73,7 +73,7 @@ void IMergeTreeReader::fillMissingColumns(Columns & res_columns, bool & should_e catch (Exception & e) { /// Better diagnostics. - e.addMessage("(while reading from part " + data_part->getFullPath() + ")"); + e.addMessage("(while reading from part " + data_part->data_part_storage->getFullPath() + ")"); throw; } } @@ -119,7 +119,7 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns catch (Exception & e) { /// Better diagnostics. - e.addMessage("(while reading from part " + data_part->getFullPath() + ")"); + e.addMessage("(while reading from part " + data_part->data_part_storage->getFullPath() + ")"); throw; } } @@ -198,7 +198,7 @@ void IMergeTreeReader::performRequiredConversions(Columns & res_columns) const catch (Exception & e) { /// Better diagnostics. - e.addMessage("(while reading from part " + data_part->getFullPath() + ")"); + e.addMessage("(while reading from part " + data_part->data_part_storage->getFullPath() + ")"); throw; } } diff --git a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index b4a902499db..e6afb6024d6 100644 --- a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -6,14 +6,15 @@ namespace DB { IMergedBlockOutputStream::IMergedBlockOutputStream( + DataPartStorageBuilderPtr data_part_storage_builder_, const MergeTreeDataPartPtr & data_part, const StorageMetadataPtr & metadata_snapshot_, const NamesAndTypesList & columns_list, bool reset_columns_) : storage(data_part->storage) , metadata_snapshot(metadata_snapshot_) - , volume(data_part->volume) - , part_path(data_part->isStoredOnDisk() ? data_part->getFullRelativePath() : "") + , data_part_storage_builder(std::move(data_part_storage_builder_)) + //, part_path(data_part->isStoredOnDisk() ? data_part->getFullRelativePath() : "") , reset_columns(reset_columns_) { if (reset_columns) diff --git a/src/Storages/MergeTree/IMergedBlockOutputStream.h b/src/Storages/MergeTree/IMergedBlockOutputStream.h index a7c25edabd4..677acf31fbe 100644 --- a/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -12,6 +12,7 @@ class IMergedBlockOutputStream { public: IMergedBlockOutputStream( + DataPartStorageBuilderPtr data_part_storage_builder_, const MergeTreeDataPartPtr & data_part, const StorageMetadataPtr & metadata_snapshot_, const NamesAndTypesList & columns_list, @@ -44,9 +45,7 @@ protected: const MergeTreeData & storage; StorageMetadataPtr metadata_snapshot; - VolumePtr volume; - String part_path; - + DataPartStorageBuilderPtr data_part_storage_builder; IMergeTreeDataPart::MergeTreeWriterPtr writer; bool reset_columns = false; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 38639328c07..1ca3caf1bd1 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -4200,6 +4200,12 @@ ReservationPtr MergeTreeData::reserveSpace(UInt64 expected_size, SpacePtr space) return checkAndReturnReservation(expected_size, std::move(reservation)); } +ReservationPtr MergeTreeData::reserveSpace(UInt64 expected_size, const DataPartStorageBuilderPtr & data_part_storage_builder) const +{ + expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size); + return data_part_storage_builder->reserve(expected_size); +} + ReservationPtr MergeTreeData::tryReserveSpace(UInt64 expected_size, SpacePtr space) { expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 23b1cecb32a..330cfc1e1da 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -409,6 +409,7 @@ public: SelectQueryInfo & info) const override; ReservationPtr reserveSpace(UInt64 expected_size, VolumePtr & volume) const; + ReservationPtr reserveSpace(UInt64 expected_size, const DataPartStorageBuilderPtr & data_part_storage_builder) const; static bool partsContainSameProjections(const DataPartPtr & left, const DataPartPtr & right); diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 00b5ee2e483..a5e5291e989 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -53,6 +53,7 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader( } IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( + DataPartStorageBuilderPtr data_part_storage_builder, const NamesAndTypesList & columns_list, const StorageMetadataPtr & metadata_snapshot, const std::vector & indices_to_recalc, @@ -69,7 +70,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( { return *getColumnPosition(lhs.name) < *getColumnPosition(rhs.name); }); return std::make_unique( - shared_from_this(), ordered_columns_list, metadata_snapshot, + shared_from_this(), std::move(data_part_storage_builder), ordered_columns_list, metadata_snapshot, indices_to_recalc, index_granularity_info.marks_file_extension, default_codec_, writer_settings, computed_index_granularity); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 48d5ef3cb73..87c9f12a328 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -45,6 +45,7 @@ public: const ReadBufferFromFileBase::ProfileCallback & profile_callback) const override; MergeTreeWriterPtr getWriter( + DataPartStorageBuilderPtr data_part_storage_builder, const NamesAndTypesList & columns_list, const StorageMetadataPtr & metadata_snapshot, const std::vector & indices_to_recalc, diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp index 2925f244d59..4fd3d00e44f 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp @@ -54,6 +54,7 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartInMemory::getReader( } IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartInMemory::getWriter( + DataPartStorageBuilderPtr, const NamesAndTypesList & columns_list, const StorageMetadataPtr & metadata_snapshot, const std::vector & /* indices_to_recalc */, diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.h b/src/Storages/MergeTree/MergeTreeDataPartInMemory.h index d015bb01b64..b9fe94b2b37 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.h +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.h @@ -34,6 +34,7 @@ public: const ReadBufferFromFileBase::ProfileCallback & profile_callback) const override; MergeTreeWriterPtr getWriter( + DataPartStorageBuilderPtr, const NamesAndTypesList & columns_list, const StorageMetadataPtr & metadata_snapshot, const std::vector & indices_to_recalc, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index eec863510eb..63fbeb1a3e2 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -55,6 +55,7 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartWide::getReader( } IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter( + DataPartStorageBuilderPtr data_part_storage_builder, const NamesAndTypesList & columns_list, const StorageMetadataPtr & metadata_snapshot, const std::vector & indices_to_recalc, @@ -63,7 +64,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter( const MergeTreeIndexGranularity & computed_index_granularity) const { return std::make_unique( - shared_from_this(), columns_list, metadata_snapshot, indices_to_recalc, + shared_from_this(), data_part_storage_builder, columns_list, metadata_snapshot, indices_to_recalc, index_granularity_info.marks_file_extension, default_codec_, writer_settings, computed_index_granularity); } @@ -144,7 +145,7 @@ void MergeTreeDataPartWide::loadIndexGranularity() bool MergeTreeDataPartWide::isStoredOnRemoteDisk() const { - return volume->getDisk()->isRemote(); + return data_part_storage->isStoredOnRemoteDisk(); } MergeTreeDataPartWide::~MergeTreeDataPartWide() diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.h b/src/Storages/MergeTree/MergeTreeDataPartWide.h index 81f9b65a5bc..b6dd6bdfced 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -39,6 +39,7 @@ public: const ReadBufferFromFileBase::ProfileCallback & profile_callback) const override; MergeTreeWriterPtr getWriter( + DataPartStorageBuilderPtr data_part_storage_builder, const NamesAndTypesList & columns_list, const StorageMetadataPtr & metadata_snapshot, const std::vector & indices_to_recalc, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index d7b8f2c4165..e2224603fce 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -11,6 +11,7 @@ namespace ErrorCodes MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( const MergeTreeData::DataPartPtr & data_part_, + DataPartStorageBuilderPtr data_part_storage_builder_, const NamesAndTypesList & columns_list_, const StorageMetadataPtr & metadata_snapshot_, const std::vector & indices_to_recalc_, @@ -18,18 +19,16 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) - : MergeTreeDataPartWriterOnDisk(data_part_, columns_list_, metadata_snapshot_, + : MergeTreeDataPartWriterOnDisk(data_part_, std::move(data_part_storage_builder_), columns_list_, metadata_snapshot_, indices_to_recalc_, marks_file_extension_, default_codec_, settings_, index_granularity_) - , plain_file(data_part->volume->getDisk()->writeFile( - part_path + MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION, - settings.max_compress_block_size, - WriteMode::Rewrite)) + , plain_file(data_part_storage_builder->writeFile( + MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION, + settings.max_compress_block_size)) , plain_hashing(*plain_file) - , marks_file(data_part->volume->getDisk()->writeFile( - part_path + MergeTreeDataPartCompact::DATA_FILE_NAME + marks_file_extension_, - 4096, - WriteMode::Rewrite)) + , marks_file(data_part_storage_builder->writeFile( + MergeTreeDataPartCompact::DATA_FILE_NAME + marks_file_extension_, + 4096)) , marks(*marks_file) { const auto & storage_columns = metadata_snapshot->getColumns(); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index cc33d8404c2..dd098b125cd 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -10,6 +10,7 @@ class MergeTreeDataPartWriterCompact : public MergeTreeDataPartWriterOnDisk public: MergeTreeDataPartWriterCompact( const MergeTreeData::DataPartPtr & data_part, + DataPartStorageBuilderPtr data_part_storage_builder_, const NamesAndTypesList & columns_list, const StorageMetadataPtr & metadata_snapshot_, const std::vector & indices_to_recalc, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.cpp index 0c715a7c27f..e1145868ce2 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.cpp @@ -15,7 +15,7 @@ MergeTreeDataPartWriterInMemory::MergeTreeDataPartWriterInMemory( const NamesAndTypesList & columns_list_, const StorageMetadataPtr & metadata_snapshot_, const MergeTreeWriterSettings & settings_) - : IMergeTreeDataPartWriter(part_, columns_list_, metadata_snapshot_, settings_) + : IMergeTreeDataPartWriter(part_, nullptr, columns_list_, metadata_snapshot_, settings_) , part_in_memory(part_) {} void MergeTreeDataPartWriterInMemory::write( diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index 8dca93f574f..8b2f195e4bb 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -41,7 +41,7 @@ void MergeTreeDataPartWriterOnDisk::Stream::sync() const MergeTreeDataPartWriterOnDisk::Stream::Stream( const String & escaped_column_name_, - DiskPtr disk_, + const DataPartStorageBuilderPtr & data_part_storage_builder, const String & data_path_, const std::string & data_file_extension_, const std::string & marks_path_, @@ -51,11 +51,11 @@ MergeTreeDataPartWriterOnDisk::Stream::Stream( escaped_column_name(escaped_column_name_), data_file_extension{data_file_extension_}, marks_file_extension{marks_file_extension_}, - plain_file(disk_->writeFile(data_path_ + data_file_extension, max_compress_block_size_, WriteMode::Rewrite)), + plain_file(data_part_storage_builder->writeFile(data_path_ + data_file_extension, max_compress_block_size_)), plain_hashing(*plain_file), compressed_buf(plain_hashing, compression_codec_, max_compress_block_size_), compressed(compressed_buf), - marks_file(disk_->writeFile(marks_path_ + marks_file_extension, 4096, WriteMode::Rewrite)), marks(*marks_file) + marks_file(data_part_storage_builder->writeFile(marks_path_ + marks_file_extension, 4096)), marks(*marks_file) { } @@ -76,6 +76,7 @@ void MergeTreeDataPartWriterOnDisk::Stream::addToChecksums(MergeTreeData::DataPa MergeTreeDataPartWriterOnDisk::MergeTreeDataPartWriterOnDisk( const MergeTreeData::DataPartPtr & data_part_, + DataPartStorageBuilderPtr data_part_storage_builder_, const NamesAndTypesList & columns_list_, const StorageMetadataPtr & metadata_snapshot_, const MergeTreeIndices & indices_to_recalc_, @@ -83,10 +84,10 @@ MergeTreeDataPartWriterOnDisk::MergeTreeDataPartWriterOnDisk( const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) - : IMergeTreeDataPartWriter(data_part_, + : IMergeTreeDataPartWriter(data_part_, std::move(data_part_storage_builder_), columns_list_, metadata_snapshot_, settings_, index_granularity_) , skip_indices(indices_to_recalc_) - , part_path(data_part_->getFullRelativePath()) + //, part_path(data_part_->getFullRelativePath()) , marks_file_extension(marks_file_extension_) , default_codec(default_codec_) , compute_granularity(index_granularity.empty()) @@ -94,9 +95,8 @@ MergeTreeDataPartWriterOnDisk::MergeTreeDataPartWriterOnDisk( if (settings.blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); - auto disk = data_part->volume->getDisk(); - if (!disk->exists(part_path)) - disk->createDirectories(part_path); + if (!data_part_storage_builder->exists()) + data_part_storage_builder->createDirectories(); if (settings.rewrite_primary_key) initPrimaryIndex(); @@ -156,7 +156,7 @@ void MergeTreeDataPartWriterOnDisk::initPrimaryIndex() { if (metadata_snapshot->hasPrimaryKey()) { - index_file_stream = data_part->volume->getDisk()->writeFile(part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); + index_file_stream = data_part_storage_builder->writeFile("primary.idx", DBMS_DEFAULT_BUFFER_SIZE); index_stream = std::make_unique(*index_file_stream); } } @@ -169,9 +169,9 @@ void MergeTreeDataPartWriterOnDisk::initSkipIndices() skip_indices_streams.emplace_back( std::make_unique( stream_name, - data_part->volume->getDisk(), - part_path + stream_name, index_helper->getSerializedFileExtension(), - part_path + stream_name, marks_file_extension, + data_part_storage_builder, + stream_name, index_helper->getSerializedFileExtension(), + stream_name, marks_file_extension, default_codec, settings.max_compress_block_size)); skip_indices_aggregators.push_back(index_helper->createIndexAggregator()); skip_index_accumulated_marks.push_back(0); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h index 5af8cbc1650..b8fae091b8c 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h @@ -49,7 +49,7 @@ public: { Stream( const String & escaped_column_name_, - DiskPtr disk_, + const DataPartStorageBuilderPtr & data_part_storage_builder, const String & data_path_, const std::string & data_file_extension_, const std::string & marks_path_, @@ -86,6 +86,7 @@ public: MergeTreeDataPartWriterOnDisk( const MergeTreeData::DataPartPtr & data_part_, + DataPartStorageBuilderPtr data_part_storage_builder_, const NamesAndTypesList & columns_list, const StorageMetadataPtr & metadata_snapshot_, const std::vector & indices_to_recalc, @@ -127,7 +128,7 @@ protected: const MergeTreeIndices skip_indices; - const String part_path; + //const String part_path; const String marks_file_extension; const CompressionCodecPtr default_codec; diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index a3eec3e54bc..142d3b911a7 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -71,6 +71,7 @@ Granules getGranulesToWrite(const MergeTreeIndexGranularity & index_granularity, MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( const MergeTreeData::DataPartPtr & data_part_, + DataPartStorageBuilderPtr data_part_storage_builder_, const NamesAndTypesList & columns_list_, const StorageMetadataPtr & metadata_snapshot_, const std::vector & indices_to_recalc_, @@ -78,7 +79,7 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) - : MergeTreeDataPartWriterOnDisk(data_part_, columns_list_, metadata_snapshot_, + : MergeTreeDataPartWriterOnDisk(data_part_, std::move(data_part_storage_builder_), columns_list_, metadata_snapshot_, indices_to_recalc_, marks_file_extension_, default_codec_, settings_, index_granularity_) { @@ -111,9 +112,9 @@ void MergeTreeDataPartWriterWide::addStreams( column_streams[stream_name] = std::make_unique( stream_name, - data_part->volume->getDisk(), - part_path + stream_name, DATA_FILE_EXTENSION, - part_path + stream_name, marks_file_extension, + data_part_storage_builder, + stream_name, DATA_FILE_EXTENSION, + stream_name, marks_file_extension, compression_codec, settings.max_compress_block_size); }; @@ -408,17 +409,16 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai if (!type->isValueRepresentedByNumber() || type->haveSubtypes() || serialization->getKind() != ISerialization::Kind::DEFAULT) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot validate column of non fixed type {}", type->getName()); - auto disk = data_part->volume->getDisk(); String escaped_name = escapeForFileName(name); - String mrk_path = part_path + escaped_name + marks_file_extension; - String bin_path = part_path + escaped_name + DATA_FILE_EXTENSION; + String mrk_path = escaped_name + marks_file_extension; + String bin_path = escaped_name + DATA_FILE_EXTENSION; /// Some columns may be removed because of ttl. Skip them. - if (!disk->exists(mrk_path)) + if (!data_part_storage_builder->exists(mrk_path)) return; - auto mrk_in = disk->readFile(mrk_path); - DB::CompressedReadBufferFromFile bin_in(disk->readFile(bin_path)); + auto mrk_in = data_part_storage_builder->readFile(mrk_path, {}, std::nullopt, std::nullopt); + DB::CompressedReadBufferFromFile bin_in(data_part_storage_builder->readFile(bin_path, {}, std::nullopt, std::nullopt)); bool must_be_last = false; UInt64 offset_in_compressed_file = 0; UInt64 offset_in_decompressed_block = 0; @@ -469,7 +469,7 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai if (index_granularity_rows != index_granularity.getMarkRows(mark_num)) throw Exception( ErrorCodes::LOGICAL_ERROR, "Incorrect mark rows for part {} for mark #{} (compressed offset {}, decompressed offset {}), in-memory {}, on disk {}, total marks {}", - data_part->getFullPath(), mark_num, offset_in_compressed_file, offset_in_decompressed_block, index_granularity.getMarkRows(mark_num), index_granularity_rows, index_granularity.getMarksCount()); + data_part_storage_builder->getFullPath(), mark_num, offset_in_compressed_file, offset_in_decompressed_block, index_granularity.getMarkRows(mark_num), index_granularity_rows, index_granularity.getMarksCount()); auto column = type->createColumn(); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index b82fcd652ae..a3517f3aa88 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -19,6 +19,7 @@ class MergeTreeDataPartWriterWide : public MergeTreeDataPartWriterOnDisk public: MergeTreeDataPartWriterWide( const MergeTreeData::DataPartPtr & data_part, + DataPartStorageBuilderPtr data_part_storage_builder_, const NamesAndTypesList & columns_list, const StorageMetadataPtr & metadata_snapshot, const std::vector & indices_to_recalc, diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index a206d7a5f55..4506abc3c1c 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -367,9 +367,15 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPart( ReservationPtr reservation = data.reserveSpacePreferringTTLRules(metadata_snapshot, expected_size, move_ttl_infos, time(nullptr), 0, true); VolumePtr volume = data.getStoragePolicy()->getVolume(0); + VolumePtr data_part_volume = createVolumeFromReservation(reservation, volume); auto data_part_storage = std::make_shared( - createVolumeFromReservation(reservation, volume), + data_part_volume, + data.relative_data_path, + TMP_PREFIX + part_name); + + auto data_part_storage_builder = std::make_shared( + data_part_volume, data.relative_data_path, TMP_PREFIX + part_name); @@ -399,15 +405,15 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPart( if (new_data_part->isStoredOnDisk()) { /// The name could be non-unique in case of stale files from previous runs. - String full_path = new_data_part->getFullRelativePath(); + String full_path = new_data_part->data_part_storage->getFullPath(); - if (new_data_part->volume->getDisk()->exists(full_path)) + if (new_data_part->data_part_storage->exists()) { - LOG_WARNING(log, "Removing old temporary directory {}", fullPath(new_data_part->volume->getDisk(), full_path)); - new_data_part->volume->getDisk()->removeRecursive(full_path); + LOG_WARNING(log, "Removing old temporary directory {}", new_data_part->data_part_storage->getFullPath()); + data_part_volume->getDisk()->removeRecursive(full_path); } - const auto disk = new_data_part->volume->getDisk(); + const auto disk = data_part_volume->getDisk(); disk->createDirectories(full_path); if (data.getSettings()->fsync_part_directory) @@ -437,7 +443,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPart( auto compression_codec = data.getContext()->chooseCompressionCodec(0, 0); const auto & index_factory = MergeTreeIndexFactory::instance(); - auto out = std::make_unique(new_data_part, metadata_snapshot, columns, + auto out = std::make_unique(new_data_part, data_part_storage_builder, metadata_snapshot, columns, index_factory.getMany(metadata_snapshot->getSecondaryIndices()), compression_codec); out->writeWithPermutation(block, perm_ptr); @@ -447,7 +453,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPart( auto projection_block = projection.calculate(block, context); if (projection_block.rows()) { - auto proj_temp_part = writeProjectionPart(data, log, projection_block, projection, new_data_part.get()); + auto proj_temp_part = writeProjectionPart(data, log, projection_block, projection, data_part_storage_builder, new_data_part.get()); new_data_part->addProjectionPart(projection.name, std::move(proj_temp_part.part)); for (auto & stream : proj_temp_part.streams) temp_part.streams.emplace_back(std::move(stream)); @@ -481,6 +487,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( const String & part_name, MergeTreeDataPartType part_type, const String & relative_path, + const DataPartStorageBuilderPtr & data_part_storage_builder, bool is_temp, const IMergeTreeDataPart * parent_part, const MergeTreeData & data, @@ -491,13 +498,15 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( TemporaryPart temp_part; const StorageMetadataPtr & metadata_snapshot = projection.metadata; MergeTreePartInfo new_part_info("all", 0, 0, 0); + auto projection_part_storage = parent_part->data_part_storage->getProjection(relative_path); auto new_data_part = data.createPart( part_name, part_type, new_part_info, - parent_part->volume, - relative_path, + projection_part_storage, parent_part); + + auto projection_part_storage_builder = data_part_storage_builder->getProjection(relative_path); new_data_part->is_temp = is_temp; NamesAndTypesList columns = metadata_snapshot->getColumns().getAllPhysical().filter(block.getNames()); @@ -511,15 +520,13 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( if (new_data_part->isStoredOnDisk()) { /// The name could be non-unique in case of stale files from previous runs. - String full_path = new_data_part->getFullRelativePath(); - - if (new_data_part->volume->getDisk()->exists(full_path)) + if (projection_part_storage->exists()) { - LOG_WARNING(log, "Removing old temporary directory {}", fullPath(new_data_part->volume->getDisk(), full_path)); - new_data_part->volume->getDisk()->removeRecursive(full_path); + LOG_WARNING(log, "Removing old temporary directory {}", projection_part_storage->getFullPath()); + projection_part_storage_builder->removeRecursive(); } - new_data_part->volume->getDisk()->createDirectories(full_path); + projection_part_storage_builder->createDirectories(); } /// If we need to calculate some columns to sort. @@ -563,6 +570,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( auto out = std::make_unique( new_data_part, + projection_part_storage_builder, metadata_snapshot, columns, MergeTreeIndices{}, @@ -583,7 +591,12 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( } MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPart( - MergeTreeData & data, Poco::Logger * log, Block block, const ProjectionDescription & projection, const IMergeTreeDataPart * parent_part) + MergeTreeData & data, + Poco::Logger * log, + Block block, + const ProjectionDescription & projection, + const DataPartStorageBuilderPtr & data_part_storage_builder, + const IMergeTreeDataPart * parent_part) { String part_name = projection.name; MergeTreeDataPartType part_type; @@ -596,7 +609,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPart( /// Size of part would not be greater than block.bytes() + epsilon size_t expected_size = block.bytes(); // just check if there is enough space on parent volume - data.reserveSpace(expected_size, parent_part->volume); + data.reserveSpace(expected_size, data_part_storage_builder); part_type = data.choosePartTypeOnDisk(expected_size, block.rows()); } @@ -604,6 +617,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPart( part_name, part_type, part_name + ".proj" /* relative_path */, + data_part_storage_builder, false /* is_temp */, parent_part, data, @@ -619,6 +633,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempProjectionPart( Poco::Logger * log, Block block, const ProjectionDescription & projection, + const DataPartStorageBuilderPtr & data_part_storage_builder, const IMergeTreeDataPart * parent_part, size_t block_num) { @@ -633,7 +648,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempProjectionPart( /// Size of part would not be greater than block.bytes() + epsilon size_t expected_size = block.bytes(); // just check if there is enough space on parent volume - data.reserveSpace(expected_size, parent_part->volume); + data.reserveSpace(expected_size, data_part_storage_builder); part_type = data.choosePartTypeOnDisk(expected_size, block.rows()); } @@ -641,6 +656,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempProjectionPart( part_name, part_type, part_name + ".tmp_proj" /* relative_path */, + data_part_storage_builder, true /* is_temp */, parent_part, data, @@ -654,12 +670,14 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeInMemoryProjectionP Poco::Logger * log, Block block, const ProjectionDescription & projection, + const DataPartStorageBuilderPtr & data_part_storage_builder, const IMergeTreeDataPart * parent_part) { return writeProjectionPartImpl( projection.name, MergeTreeDataPartType::IN_MEMORY, projection.name + ".proj" /* relative_path */, + data_part_storage_builder, false /* is_temp */, parent_part, data, diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.h b/src/Storages/MergeTree/MergeTreeDataWriter.h index 7b6bf8fb1db..147b38e828a 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.h +++ b/src/Storages/MergeTree/MergeTreeDataWriter.h @@ -74,6 +74,7 @@ public: Poco::Logger * log, Block block, const ProjectionDescription & projection, + const DataPartStorageBuilderPtr & data_part_storage_builder, const IMergeTreeDataPart * parent_part); /// For mutation: MATERIALIZE PROJECTION. @@ -82,6 +83,7 @@ public: Poco::Logger * log, Block block, const ProjectionDescription & projection, + const DataPartStorageBuilderPtr & data_part_storage_builder, const IMergeTreeDataPart * parent_part, size_t block_num); @@ -91,6 +93,7 @@ public: Poco::Logger * log, Block block, const ProjectionDescription & projection, + const DataPartStorageBuilderPtr & data_part_storage_builder, const IMergeTreeDataPart * parent_part); static Block mergeBlock( @@ -105,6 +108,7 @@ private: const String & part_name, MergeTreeDataPartType part_type, const String & relative_path, + const DataPartStorageBuilderPtr & data_part_storage_builder, bool is_temp, const IMergeTreeDataPart * parent_part, const MergeTreeData & data, diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index e7ead4dc8bb..c948a9bbced 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -16,14 +16,14 @@ namespace ErrorCodes } MergeTreeMarksLoader::MergeTreeMarksLoader( - DiskPtr disk_, + DataPartStoragePtr data_part_storage_, MarkCache * mark_cache_, const String & mrk_path_, size_t marks_count_, const MergeTreeIndexGranularityInfo & index_granularity_info_, bool save_marks_in_cache_, size_t columns_in_mark_) - : disk(std::move(disk_)) + : data_part_storage(std::move(data_part_storage_)) , mark_cache(mark_cache_) , mrk_path(mrk_path_) , marks_count(marks_count_) @@ -50,21 +50,23 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. MemoryTrackerBlockerInThread temporarily_disable_memory_tracker; - size_t file_size = disk->getFileSize(mrk_path); + size_t file_size = data_part_storage->getFileSize(mrk_path); size_t mark_size = index_granularity_info.getMarkSizeInBytes(columns_in_mark); size_t expected_file_size = mark_size * marks_count; if (expected_file_size != file_size) throw Exception( - "Bad size of marks file '" + fullPath(disk, mrk_path) + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), - ErrorCodes::CORRUPTED_DATA); + ErrorCodes::CORRUPTED_DATA, + "Bad size of marks file '{}': {}, must be: {}", + std::string(fs::path(data_part_storage->getFullPath()) / mrk_path), + std::to_string(file_size), std::to_string(expected_file_size)); auto res = std::make_shared(marks_count * columns_in_mark); if (!index_granularity_info.is_adaptive) { /// Read directly to marks. - auto buffer = disk->readFile(mrk_path, ReadSettings().adjustBufferSize(file_size), file_size); + auto buffer = data_part_storage->readFile(mrk_path, ReadSettings().adjustBufferSize(file_size), file_size, std::nullopt); buffer->readStrict(reinterpret_cast(res->data()), file_size); if (!buffer->eof()) @@ -73,7 +75,7 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() } else { - auto buffer = disk->readFile(mrk_path, ReadSettings().adjustBufferSize(file_size), file_size); + auto buffer = data_part_storage->readFile(mrk_path, ReadSettings().adjustBufferSize(file_size), file_size, std::nullopt); size_t i = 0; while (!buffer->eof()) { diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.h b/src/Storages/MergeTree/MergeTreeMarksLoader.h index 062eb720583..3effeeec12b 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include namespace DB @@ -13,7 +13,7 @@ public: using MarksPtr = MarkCache::MappedPtr; MergeTreeMarksLoader( - DiskPtr disk_, + DataPartStoragePtr data_part_storage_, MarkCache * mark_cache_, const String & mrk_path, size_t marks_count_, @@ -26,7 +26,7 @@ public: bool initialized() const { return marks != nullptr; } private: - DiskPtr disk; + DataPartStoragePtr data_part_storage; MarkCache * mark_cache = nullptr; String mrk_path; size_t marks_count; diff --git a/src/Storages/MergeTree/MergeTreePartition.cpp b/src/Storages/MergeTree/MergeTreePartition.cpp index 4955a63387c..8cd94673e8c 100644 --- a/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/src/Storages/MergeTree/MergeTreePartition.cpp @@ -381,19 +381,19 @@ void MergeTreePartition::load(const MergeTreeData & storage, const PartMetadataM partition_key_sample.getByPosition(i).type->getDefaultSerialization()->deserializeBinary(value[i], *file); } -std::unique_ptr MergeTreePartition::store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const +std::unique_ptr MergeTreePartition::store(const MergeTreeData & storage, const DataPartStorageBuilderPtr & data_part_storage_builder, MergeTreeDataPartChecksums & checksums) const { auto metadata_snapshot = storage.getInMemoryMetadataPtr(); const auto & partition_key_sample = adjustPartitionKey(metadata_snapshot, storage.getContext()).sample_block; - return store(partition_key_sample, disk, part_path, checksums); + return store(partition_key_sample, data_part_storage_builder, checksums); } -std::unique_ptr MergeTreePartition::store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const +std::unique_ptr MergeTreePartition::store(const Block & partition_key_sample, const DataPartStorageBuilderPtr & data_part_storage_builder, MergeTreeDataPartChecksums & checksums) const { if (!partition_key_sample) return nullptr; - auto out = disk->writeFile(part_path + "partition.dat"); + auto out = data_part_storage_builder->writeFile("partition.dat", DBMS_DEFAULT_BUFFER_SIZE); HashingWriteBuffer out_hashing(*out); for (size_t i = 0; i < value.size(); ++i) { diff --git a/src/Storages/MergeTree/MergeTreePartition.h b/src/Storages/MergeTree/MergeTreePartition.h index e7328799e4c..8ff359a17db 100644 --- a/src/Storages/MergeTree/MergeTreePartition.h +++ b/src/Storages/MergeTree/MergeTreePartition.h @@ -15,8 +15,10 @@ class MergeTreeData; struct FormatSettings; struct MergeTreeDataPartChecksums; struct StorageInMemoryMetadata; +class IDataPartStorageBuilder; using StorageMetadataPtr = std::shared_ptr; +using DataPartStorageBuilderPtr = std::shared_ptr; /// This class represents a partition value of a single part and encapsulates its loading/storing logic. struct MergeTreePartition @@ -42,8 +44,8 @@ public: /// Store functions return write buffer with written but not finalized data. /// User must call finish() for returned object. - [[nodiscard]] std::unique_ptr store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const; - [[nodiscard]] std::unique_ptr store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const; + [[nodiscard]] std::unique_ptr store(const MergeTreeData & storage, const DataPartStorageBuilderPtr & data_part_storage_builder, MergeTreeDataPartChecksums & checksums) const; + [[nodiscard]] std::unique_ptr store(const Block & partition_key_sample, const DataPartStorageBuilderPtr & data_part_storage_builder, MergeTreeDataPartChecksums & checksums) const; void assign(const MergeTreePartition & other) { value = other.value; } diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index b943c3c8718..2281bc3e57f 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -35,9 +35,9 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( settings_, avg_value_size_hints_) , marks_loader( - data_part->volume->getDisk(), + data_part->data_part_storage, mark_cache, - data_part->index_granularity_info.getMarksFilePath(data_part->getFullRelativePath() + MergeTreeDataPartCompact::DATA_FILE_NAME), + data_part->index_granularity_info.getMarksFilePath(MergeTreeDataPartCompact::DATA_FILE_NAME), data_part->getMarksCount(), data_part->index_granularity_info, settings.save_marks_in_cache, @@ -83,16 +83,17 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( if (!settings.read_settings.local_fs_buffer_size || !settings.read_settings.remote_fs_buffer_size) throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read to empty buffer."); - const String full_data_path = data_part->getFullRelativePath() + MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION; + const String path = MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION; if (uncompressed_cache) { auto buffer = std::make_unique( - fullPath(data_part->volume->getDisk(), full_data_path), - [this, full_data_path]() + std::string(fs::path(data_part->data_part_storage->getFullPath()) / path), + [this, path]() { - return data_part->volume->getDisk()->readFile( - full_data_path, - settings.read_settings); + return data_part->data_part_storage->readFile( + path, + settings.read_settings, + std::nullopt, std::nullopt); }, uncompressed_cache, /* allow_different_codecs = */ true); @@ -111,9 +112,10 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( { auto buffer = std::make_unique( - data_part->volume->getDisk()->readFile( - full_data_path, - settings.read_settings), + data_part->data_part_storage->readFile( + path, + settings.read_settings, + std::nullopt, std::nullopt), /* allow_different_codecs = */ true); if (profile_callback_) diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp index b337bd62dd3..d1e9ed2e5f1 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -15,7 +15,7 @@ namespace ErrorCodes } MergeTreeReaderStream::MergeTreeReaderStream( - DiskPtr disk_, + DataPartStoragePtr data_part_storage_, const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, const MarkRanges & all_mark_ranges, const MergeTreeReaderSettings & settings, @@ -23,7 +23,7 @@ MergeTreeReaderStream::MergeTreeReaderStream( UncompressedCache * uncompressed_cache, size_t file_size_, const MergeTreeIndexGranularityInfo * index_granularity_info_, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type) - : disk(std::move(disk_)) + : data_part_storage(std::move(data_part_storage_)) , path_prefix(path_prefix_) , data_file_extension(data_file_extension_) , marks_count(marks_count_) @@ -31,7 +31,7 @@ MergeTreeReaderStream::MergeTreeReaderStream( , mark_cache(mark_cache_) , save_marks_in_cache(settings.save_marks_in_cache) , index_granularity_info(index_granularity_info_) - , marks_loader(disk, mark_cache, index_granularity_info->getMarksFilePath(path_prefix), + , marks_loader(data_part_storage, mark_cache, index_granularity_info->getMarksFilePath(path_prefix), marks_count, *index_granularity_info, save_marks_in_cache) { /// Compute the size of the buffer. @@ -68,13 +68,13 @@ MergeTreeReaderStream::MergeTreeReaderStream( if (uncompressed_cache) { auto buffer = std::make_unique( - fullPath(disk, path_prefix + data_file_extension), + std::string(fs::path(data_part_storage->getFullPath()) / (path_prefix + data_file_extension)), [this, estimated_sum_mark_range_bytes, read_settings]() { - return disk->readFile( + return data_part_storage->readFile( path_prefix + data_file_extension, read_settings, - estimated_sum_mark_range_bytes); + estimated_sum_mark_range_bytes, std::nullopt); }, uncompressed_cache); @@ -91,10 +91,11 @@ MergeTreeReaderStream::MergeTreeReaderStream( else { auto buffer = std::make_unique( - disk->readFile( + data_part_storage->readFile( path_prefix + data_file_extension, read_settings, - estimated_sum_mark_range_bytes)); + estimated_sum_mark_range_bytes, + std::nullopt)); if (profile_callback) buffer->setProfileCallback(profile_callback, clock_type); diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.h b/src/Storages/MergeTree/MergeTreeReaderStream.h index 0b6b86e7f82..9a3de3ef967 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -18,7 +18,7 @@ class MergeTreeReaderStream { public: MergeTreeReaderStream( - DiskPtr disk_, + DataPartStoragePtr data_part_storage_, const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, const MarkRanges & all_mark_ranges, const MergeTreeReaderSettings & settings_, @@ -42,7 +42,7 @@ public: private: size_t getRightOffset(size_t right_mark_non_included); - DiskPtr disk; + DataPartStoragePtr data_part_storage; std::string path_prefix; std::string data_file_extension; diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 7d7975e0bc0..c168e5867d4 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -47,7 +47,6 @@ MergeTreeReaderWide::MergeTreeReaderWide( { try { - disk = data_part->volume->getDisk(); for (const NameAndTypePair & column : columns) { auto column_from_part = getColumnFromPart(column); @@ -74,7 +73,7 @@ size_t MergeTreeReaderWide::readRows( std::unordered_map caches; std::unordered_set prefetched_streams; - if (disk->isRemote() ? settings.read_settings.remote_fs_prefetch : settings.read_settings.local_fs_prefetch) + if (data_part->data_part_storage->isStoredOnRemoteDisk() ? settings.read_settings.remote_fs_prefetch : settings.read_settings.local_fs_prefetch) { /// Request reading of data in advance, /// so if reading can be asynchronous, it will also be performed in parallel for all columns. @@ -147,7 +146,7 @@ size_t MergeTreeReaderWide::readRows( storage.reportBrokenPart(data_part); /// Better diagnostics. - e.addMessage("(while reading from part " + data_part->getFullPath() + " " + e.addMessage("(while reading from part " + data_part->data_part_storage->getFullPath() + " " "from mark " + toString(from_mark) + " " "with max_rows_to_read = " + toString(max_rows_to_read) + ")"); throw; @@ -181,7 +180,7 @@ void MergeTreeReaderWide::addStreams(const NameAndTypePair & name_and_type, return; streams.emplace(stream_name, std::make_unique( - disk, data_part->getFullRelativePath() + stream_name, DATA_FILE_EXTENSION, + data_part->data_part_storage, stream_name, DATA_FILE_EXTENSION, data_part->getMarksCount(), all_mark_ranges, settings, mark_cache, uncompressed_cache, data_part->getFileSizeOrZero(stream_name + DATA_FILE_EXTENSION), &data_part->index_granularity_info, diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.h b/src/Storages/MergeTree/MergeTreeReaderWide.h index 41219560ecc..7bb1ccfd173 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.h +++ b/src/Storages/MergeTree/MergeTreeReaderWide.h @@ -37,7 +37,6 @@ public: private: FileStreams streams; - DiskPtr disk; void addStreams(const NameAndTypePair & name_and_type, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type); diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index f94c89e20bd..fc0bc635df4 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -14,13 +14,14 @@ namespace ErrorCodes MergedBlockOutputStream::MergedBlockOutputStream( const MergeTreeDataPartPtr & data_part, + DataPartStorageBuilderPtr data_part_storage_builder_, const StorageMetadataPtr & metadata_snapshot_, const NamesAndTypesList & columns_list_, const MergeTreeIndices & skip_indices, CompressionCodecPtr default_codec_, bool reset_columns_, bool blocks_are_granules_size) - : IMergedBlockOutputStream(data_part, metadata_snapshot_, columns_list_, reset_columns_) + : IMergedBlockOutputStream(std::move(data_part_storage_builder_), data_part, metadata_snapshot_, columns_list_, reset_columns_) , columns_list(columns_list_) , default_codec(default_codec_) { @@ -31,10 +32,10 @@ MergedBlockOutputStream::MergedBlockOutputStream( /* rewrite_primary_key = */ true, blocks_are_granules_size); - if (!part_path.empty()) - volume->getDisk()->createDirectories(part_path); + // if (!part_path.empty()) + // volume->getDisk()->createDirectories(part_path); - writer = data_part->getWriter(columns_list, metadata_snapshot, skip_indices, default_codec, writer_settings); + writer = data_part->getWriter(std::move(data_part_storage_builder), columns_list, metadata_snapshot, skip_indices, default_codec, writer_settings); } /// If data is pre-sorted. @@ -55,13 +56,15 @@ struct MergedBlockOutputStream::Finalizer::Impl { IMergeTreeDataPartWriter & writer; MergeTreeData::MutableDataPartPtr part; + DataPartStorageBuilderPtr data_part_storage_builder; NameSet files_to_remove_after_finish; std::vector> written_files; bool sync; - Impl(IMergeTreeDataPartWriter & writer_, MergeTreeData::MutableDataPartPtr part_, const NameSet & files_to_remove_after_finish_, bool sync_) - : writer(writer_) + Impl(IMergeTreeDataPartWriter & writer_, MergeTreeData::MutableDataPartPtr part_, DataPartStorageBuilderPtr data_part_storage_builder_, const NameSet & files_to_remove_after_finish_, bool sync_) + : writer(writer_) , part(std::move(part_)) + , data_part_storage_builder(std::move(data_part_storage_builder_)) , files_to_remove_after_finish(files_to_remove_after_finish_) , sync(sync_) {} @@ -79,9 +82,8 @@ void MergedBlockOutputStream::Finalizer::Impl::finish() { writer.finish(sync); - auto disk = part->volume->getDisk(); for (const auto & file_name: files_to_remove_after_finish) - disk->removeFile(part->getFullRelativePath() + file_name); + data_part_storage_builder->removeFile(file_name); for (auto & file : written_files) { @@ -154,7 +156,7 @@ MergedBlockOutputStream::Finalizer MergedBlockOutputStream::finalizePartAsync( new_part->setSerializationInfos(serialization_infos); } - auto finalizer = std::make_unique(*writer, new_part, files_to_remove_after_sync, sync); + auto finalizer = std::make_unique(*writer, new_part, data_part_storage_builder, files_to_remove_after_sync, sync); if (new_part->isStoredOnDisk()) finalizer->written_files = finalizePartOnDisk(new_part, checksums); @@ -181,7 +183,7 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis { if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING || isCompactPart(new_part)) { - auto count_out = volume->getDisk()->writeFile(part_path + "count.txt", 4096); + auto count_out = data_part_storage_builder->writeFile("count.txt", 4096); HashingWriteBuffer count_out_hashing(*count_out); writeIntText(rows_count, count_out_hashing); count_out_hashing.next(); @@ -195,7 +197,7 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis { if (new_part->uuid != UUIDHelpers::Nil) { - auto out = volume->getDisk()->writeFile(fs::path(part_path) / IMergeTreeDataPart::UUID_FILE_NAME, 4096); + auto out = data_part_storage_builder->writeFile(IMergeTreeDataPart::UUID_FILE_NAME, 4096); HashingWriteBuffer out_hashing(*out); writeUUIDText(new_part->uuid, out_hashing); checksums.files[IMergeTreeDataPart::UUID_FILE_NAME].file_size = out_hashing.count(); @@ -206,12 +208,12 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { - if (auto file = new_part->partition.store(storage, volume->getDisk(), part_path, checksums)) + if (auto file = new_part->partition.store(storage, data_part_storage_builder, checksums)) written_files.emplace_back(std::move(file)); if (new_part->minmax_idx->initialized) { - auto files = new_part->minmax_idx->store(storage, volume->getDisk(), part_path, checksums); + auto files = new_part->minmax_idx->store(storage, data_part_storage_builder, checksums); for (auto & file : files) written_files.emplace_back(std::move(file)); } @@ -221,7 +223,7 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis } { - auto count_out = volume->getDisk()->writeFile(fs::path(part_path) / "count.txt", 4096); + auto count_out = data_part_storage_builder->writeFile("count.txt", 4096); HashingWriteBuffer count_out_hashing(*count_out); writeIntText(rows_count, count_out_hashing); count_out_hashing.next(); @@ -235,7 +237,7 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis if (!new_part->ttl_infos.empty()) { /// Write a file with ttl infos in json format. - auto out = volume->getDisk()->writeFile(fs::path(part_path) / "ttl.txt", 4096); + auto out = data_part_storage_builder->writeFile("ttl.txt", 4096); HashingWriteBuffer out_hashing(*out); new_part->ttl_infos.write(out_hashing); checksums.files["ttl.txt"].file_size = out_hashing.count(); @@ -246,7 +248,7 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis if (!new_part->getSerializationInfos().empty()) { - auto out = volume->getDisk()->writeFile(part_path + IMergeTreeDataPart::SERIALIZATION_FILE_NAME, 4096); + auto out = data_part_storage_builder->writeFile(IMergeTreeDataPart::SERIALIZATION_FILE_NAME, 4096); HashingWriteBuffer out_hashing(*out); new_part->getSerializationInfos().writeJSON(out_hashing); checksums.files[IMergeTreeDataPart::SERIALIZATION_FILE_NAME].file_size = out_hashing.count(); @@ -257,7 +259,7 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis { /// Write a file with a description of columns. - auto out = volume->getDisk()->writeFile(fs::path(part_path) / "columns.txt", 4096); + auto out = data_part_storage_builder->writeFile("columns.txt", 4096); new_part->getColumns().writeText(*out); out->preFinalize(); written_files.emplace_back(std::move(out)); @@ -265,7 +267,7 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis if (default_codec != nullptr) { - auto out = volume->getDisk()->writeFile(part_path + IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096); + auto out = data_part_storage_builder->writeFile(IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096); DB::writeText(queryToString(default_codec->getFullCodecDesc()), *out); out->preFinalize(); written_files.emplace_back(std::move(out)); @@ -278,7 +280,7 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis { /// Write file with checksums. - auto out = volume->getDisk()->writeFile(fs::path(part_path) / "checksums.txt", 4096); + auto out = data_part_storage_builder->writeFile("checksums.txt", 4096); checksums.write(*out); out->preFinalize(); written_files.emplace_back(std::move(out)); diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.h b/src/Storages/MergeTree/MergedBlockOutputStream.h index c17cfd22cd8..91a76bf260a 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -15,6 +15,7 @@ class MergedBlockOutputStream final : public IMergedBlockOutputStream public: MergedBlockOutputStream( const MergeTreeDataPartPtr & data_part, + DataPartStorageBuilderPtr data_part_storage_builder_, const StorageMetadataPtr & metadata_snapshot_, const NamesAndTypesList & columns_list_, const MergeTreeIndices & skip_indices, From b4cd8561a5e1d12621e73d21bbc3cc5cc8d31ef8 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 13 Apr 2022 16:06:19 +0800 Subject: [PATCH 0048/1954] wip --- src/Disks/IDiskRemote.cpp | 2 +- ...chronousReadIndirectBufferFromRemoteFS.cpp | 2 +- src/Disks/IO/ThreadPoolRemoteFSReader.cpp | 12 +- src/Disks/IO/ThreadPoolRemoteFSReader.h | 17 +- .../HDFS/AsynchronousReadBufferFromHDFS.cpp | 243 ++++++++++++++++++ .../HDFS/AsynchronousReadBufferFromHDFS.h | 49 ++++ src/Storages/HDFS/ReadBufferFromHDFS.cpp | 10 + src/Storages/HDFS/ReadBufferFromHDFS.h | 5 + 8 files changed, 324 insertions(+), 16 deletions(-) create mode 100644 src/Storages/HDFS/AsynchronousReadBufferFromHDFS.cpp create mode 100644 src/Storages/HDFS/AsynchronousReadBufferFromHDFS.h diff --git a/src/Disks/IDiskRemote.cpp b/src/Disks/IDiskRemote.cpp index ead951084ad..405c0a84fc0 100644 --- a/src/Disks/IDiskRemote.cpp +++ b/src/Disks/IDiskRemote.cpp @@ -645,7 +645,7 @@ AsynchronousReaderPtr IDiskRemote::getThreadPoolReader() { constexpr size_t pool_size = 50; constexpr size_t queue_size = 1000000; - static AsynchronousReaderPtr reader = std::make_shared(pool_size, queue_size); + static AsynchronousReaderPtr reader = std::make_shared>(pool_size, queue_size); return reader; } diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp index e693a8e9ea8..665c3c67126 100644 --- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp @@ -101,7 +101,7 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::hasPendingDataToRead() std::future AsynchronousReadIndirectBufferFromRemoteFS::readInto(char * data, size_t size) { IAsynchronousReader::Request request; - request.descriptor = std::make_shared(impl); + request.descriptor = std::make_shared>(impl); request.buf = data; request.size = size; request.offset = file_offset_of_buffer_end; diff --git a/src/Disks/IO/ThreadPoolRemoteFSReader.cpp b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp index b1ae42d03d6..e63008b3906 100644 --- a/src/Disks/IO/ThreadPoolRemoteFSReader.cpp +++ b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp @@ -27,20 +27,22 @@ namespace CurrentMetrics namespace DB { - -ReadBufferFromRemoteFSGather::ReadResult ThreadPoolRemoteFSReader::RemoteFSFileDescriptor::readInto(char * data, size_t size, size_t offset, size_t ignore) +template +IAsynchronousReader::Result RemoteFSFileDescriptor::readInto(char * data, size_t size, size_t offset, size_t ignore) { return reader->readInto(data, size, offset, ignore); } -ThreadPoolRemoteFSReader::ThreadPoolRemoteFSReader(size_t pool_size, size_t queue_size_) +template +ThreadPoolRemoteFSReader::ThreadPoolRemoteFSReader(size_t pool_size, size_t queue_size_) : pool(pool_size, pool_size, queue_size_) { } -std::future ThreadPoolRemoteFSReader::submit(Request request) +template +std::future ThreadPoolRemoteFSReader::submit(Request request) { ThreadGroupStatusPtr running_group = CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup() ? CurrentThread::get().getThreadGroup() @@ -65,7 +67,7 @@ std::future ThreadPoolRemoteFSReader::submit(Reques setThreadName("VFSRead"); CurrentMetrics::Increment metric_increment{CurrentMetrics::Read}; - auto * remote_fs_fd = assert_cast(request.descriptor.get()); + auto * remote_fs_fd = assert_cast *>(request.descriptor.get()); Stopwatch watch(CLOCK_MONOTONIC); diff --git a/src/Disks/IO/ThreadPoolRemoteFSReader.h b/src/Disks/IO/ThreadPoolRemoteFSReader.h index b2d5f11724a..c2690deacad 100644 --- a/src/Disks/IO/ThreadPoolRemoteFSReader.h +++ b/src/Disks/IO/ThreadPoolRemoteFSReader.h @@ -10,30 +10,29 @@ namespace DB { +template class ThreadPoolRemoteFSReader : public IAsynchronousReader { - -private: - ThreadPool pool; - public: ThreadPoolRemoteFSReader(size_t pool_size, size_t queue_size_); std::future submit(Request request) override; - struct RemoteFSFileDescriptor; +private: + ThreadPool pool; }; -struct ThreadPoolRemoteFSReader::RemoteFSFileDescriptor : public IFileDescriptor +template +class RemoteFSFileDescriptor : public IAsynchronousReader::IFileDescriptor { public: - explicit RemoteFSFileDescriptor(std::shared_ptr reader_) : reader(reader_) {} + explicit RemoteFSFileDescriptor(std::shared_ptr reader_) : reader(std::move(reader_)) { } - ReadBufferFromRemoteFSGather::ReadResult readInto(char * data, size_t size, size_t offset, size_t ignore = 0); + IAsynchronousReader::Result readInto(char * data, size_t size, size_t offset, size_t ignore = 0); private: - std::shared_ptr reader; + std::shared_ptr reader; }; } diff --git a/src/Storages/HDFS/AsynchronousReadBufferFromHDFS.cpp b/src/Storages/HDFS/AsynchronousReadBufferFromHDFS.cpp new file mode 100644 index 00000000000..a5d550b353c --- /dev/null +++ b/src/Storages/HDFS/AsynchronousReadBufferFromHDFS.cpp @@ -0,0 +1,243 @@ +#include "AsynchronousReadBufferFromHDFS.h" + +#if USE_HDFS +#include +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event AsynchronousReadWaitMicroseconds; +} + +namespace CurrentMetrics +{ + extern const Metric AsynchronousReadWait; +} + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NETWORK_ERROR; + extern const int CANNOT_OPEN_FILE; + extern const int CANNOT_SEEK_THROUGH_FILE; + extern const int SEEK_POSITION_OUT_OF_BOUND; + extern const int LOGICAL_ERROR; +} + +AsynchronousReadBufferFromHDFS::~AsynchronousReadBufferFromHDFS() = default; + +class AsynchronousReadBufferFromHDFS::AsynchronousReadBufferFromHDFSImpl : public BufferWithOwnMemory +{ +public: + explicit AsynchronousReadBufferFromHDFSImpl( + std::shared_ptr in_, + const std::string & hdfs_uri_, + const std::string & hdfs_file_path_, + const Poco::Util::AbstractConfiguration & config_, + size_t buf_size_, size_t read_until_position_) + : BufferWithOwnMemory(buf_size_) + , in(in_) + , hdfs_uri(hdfs_uri_) + , hdfs_file_path(hdfs_file_path_) + , builder(createHDFSBuilder(hdfs_uri_, config_)) + , read_until_position(read_until_position_) + { + fs = createHDFSFS(builder.get()); + fin = hdfsOpenFile(fs.get(), hdfs_file_path.c_str(), O_RDONLY, 0, 0, 0); + + if (fin == nullptr) + throw Exception(ErrorCodes::CANNOT_OPEN_FILE, + "Unable to open HDFS file: {}. Error: {}", + hdfs_uri + hdfs_file_path, std::string(hdfsGetLastError())); + } + + ~AsynchronousReadBufferFromHDFSImpl() override + { + hdfsCloseFile(fs.get(), fin); + } + + std::optional getTotalSize() const + { + auto * file_info = hdfsGetPathInfo(fs.get(), hdfs_file_path.c_str()); + if (!file_info) + return std::nullopt; + return file_info->mSize; + } + + bool nextImpl() override + { + if (prefetch_future.valid()) + { + /// Read request already in flight. Wait for its completion. + size_t size = 0; + size_t offset = 0; + { + Stopwatch watch; + CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait}; + auto result = prefetch_future.get(); + size = result.size; + offset = result.offset; + assert(offset < size || size == 0); + ProfileEvents::increment(ProfileEvents::AsynchronousReadWaitMicroseconds, watch.elapsedMicroseconds()); + } + prefetch_future = {}; + + if (size) + { + prefetch_buffer.swap(memory); + /// Adjust the working buffer so that it ignores `offset` bytes. + setWithBytesToIgnore(memory.data(), size, offset); + return true; + } + return false; + } + else + { + /// No pending request. Do synchronous read. + auto [size, offset] = readInto(memory.data(), memory.size()).get(); + file_offset_of_buffer_end += size; + + if (size) + { + /// Adjust the working buffer so that it ignores `offset` bytes. + setWithBytesToIgnore(memory.data(), size, offset); + return true; + } + + return false; + } + } + + off_t seek(off_t file_offset_, int whence) override + { + if (whence != SEEK_SET) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Only SEEK_SET is supported"); + + file_offset_of_buffer_end = file_offset_; + int seek_status = hdfsSeek(fs.get(), fin, file_offset_of_buffer_end); + if (seek_status != 0) + throw Exception(ErrorCodes::CANNOT_SEEK_THROUGH_FILE, "Fail to seek HDFS file: {}, error: {}", hdfs_uri, std::string(hdfsGetLastError())); + return file_offset_of_buffer_end; + } + + off_t getPosition() override + { + return file_offset_of_buffer_end; + } + + void prefetch() override + { + if (prefetch_future.valid()) + return; + + /// Will request the same amount of data that is read in nextImpl. + prefetch_buffer.resize(internal_buffer.size()); + prefetch_future = readInto(prefetch_buffer.data(), prefetch_buffer.size()); + } + + void finalize() + { + if (prefetch_future.valid()) + { + prefetch_future.wait(); + prefetch_future = {}; + } + } + + static AsynchronousReaderPtr getThreadPoolReader() + { + constexpr size_t pool_size = 50; + constexpr size_t queue_size = 1000000; + static AsynchronousReaderPtr reader = std::make_shared>(pool_size, queue_size); + return reader; + } + +private: + std::future readInto(char * data, size_t size) + { + IAsynchronousReader::Request request; + request.descriptor = std::make_shared>(in); + request.buf = data; + request.size = size; + request.offset = file_offset_of_buffer_end; + request.priority = priority; + request.ignore = 0; + return getThreadPoolReader()->submit(request); + } + + std::shared_ptr in; + Int32 priority; + Memory<> prefetch_buffer; + std::future prefetch_future; +}; + +AsynchronousReadBufferFromHDFS::AsynchronousReadBufferFromHDFS( + const String & hdfs_uri_, + const String & hdfs_file_path_, + const Poco::Util::AbstractConfiguration & config_, + size_t buf_size_, size_t read_until_position_) + : SeekableReadBufferWithSize(nullptr, 0) + , impl(std::make_unique(hdfs_uri_, hdfs_file_path_, config_, buf_size_, read_until_position_)) +{ +} + +std::optional AsynchronousReadBufferFromHDFS::getTotalSize() +{ + return impl->getTotalSize(); +} + +bool AsynchronousReadBufferFromHDFS::nextImpl() +{ + impl->position() = impl->buffer().begin() + offset(); + auto result = impl->next(); + + if (result) + BufferBase::set(impl->buffer().begin(), impl->buffer().size(), impl->offset()); /// use the buffer returned by `impl` + + return result; +} + + +off_t AsynchronousReadBufferFromHDFS::seek(off_t offset_, int whence) +{ + if (whence != SEEK_SET) + throw Exception("Only SEEK_SET mode is allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + + if (offset_ < 0) + throw Exception("Seek position is out of bounds. Offset: " + std::to_string(offset_), ErrorCodes::SEEK_POSITION_OUT_OF_BOUND); + + if (!working_buffer.empty() + && size_t(offset_) >= impl->getPosition() - working_buffer.size() + && offset_ < impl->getPosition()) + { + pos = working_buffer.end() - (impl->getPosition() - offset_); + assert(pos >= working_buffer.begin()); + assert(pos <= working_buffer.end()); + + return getPosition(); + } + + resetWorkingBuffer(); + impl->seek(offset_, whence); + return impl->getPosition(); +} + + +off_t AsynchronousReadBufferFromHDFS::getPosition() +{ + return impl->getPosition() - available(); +} + +size_t AsynchronousReadBufferFromHDFS::getFileOffsetOfBufferEnd() const +{ + return impl->getPosition(); +} + +} + +#endif diff --git a/src/Storages/HDFS/AsynchronousReadBufferFromHDFS.h b/src/Storages/HDFS/AsynchronousReadBufferFromHDFS.h new file mode 100644 index 00000000000..f542d3d3d3e --- /dev/null +++ b/src/Storages/HDFS/AsynchronousReadBufferFromHDFS.h @@ -0,0 +1,49 @@ +#pragma once + +#include + +#if USE_HDFS +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +class AsynchronousReadBufferFromHDFS : public SeekableReadBufferWithSize +{ +class AsynchronousReadBufferFromHDFSImpl; + +public: + AsynchronousReadBufferFromHDFS(const String & hdfs_uri_, const String & hdfs_file_path_, + const Poco::Util::AbstractConfiguration & config_, + size_t buf_size_ = DBMS_DEFAULT_BUFFER_SIZE, + size_t read_until_position_ = 0); + + ~AsynchronousReadBufferFromHDFS() override; + + bool nextImpl() override; + + off_t seek(off_t offset_, int whence) override; + + off_t getPosition() override; + + std::optional getTotalSize() override; + + size_t getFileOffsetOfBufferEnd() const override; + +private: + std::unique_ptr impl; +}; + +} +#endif diff --git a/src/Storages/HDFS/ReadBufferFromHDFS.cpp b/src/Storages/HDFS/ReadBufferFromHDFS.cpp index af4b324cd77..36c3626b754 100644 --- a/src/Storages/HDFS/ReadBufferFromHDFS.cpp +++ b/src/Storages/HDFS/ReadBufferFromHDFS.cpp @@ -181,6 +181,16 @@ size_t ReadBufferFromHDFS::getFileOffsetOfBufferEnd() const return impl->getPosition(); } +ReadBufferFromHDFS::ReadResult ReadBufferFromHDFS::readInto(char * data, size_t size, size_t offset, size_t ignore) +{ + set(data, size); + seek(offset, SEEK_SET); + auto result = nextImpl(); + if (result) + return {working_buffer.size(), ignore}; + return {0, 0}; +} + } #endif diff --git a/src/Storages/HDFS/ReadBufferFromHDFS.h b/src/Storages/HDFS/ReadBufferFromHDFS.h index e8cdcb27360..cff3e032e61 100644 --- a/src/Storages/HDFS/ReadBufferFromHDFS.h +++ b/src/Storages/HDFS/ReadBufferFromHDFS.h @@ -5,6 +5,7 @@ #if USE_HDFS #include #include +#include #include #include #include @@ -24,6 +25,8 @@ class ReadBufferFromHDFS : public SeekableReadBufferWithSize struct ReadBufferFromHDFSImpl; public: + using ReadResult = IAsynchronousReader::Result; + ReadBufferFromHDFS(const String & hdfs_uri_, const String & hdfs_file_path_, const Poco::Util::AbstractConfiguration & config_, size_t buf_size_ = DBMS_DEFAULT_BUFFER_SIZE, @@ -41,6 +44,8 @@ public: size_t getFileOffsetOfBufferEnd() const override; + ReadResult readInto(char * data, size_t size, size_t offset, size_t ignore = 0); + private: std::unique_ptr impl; }; From 0baa4f97ec3dc81ecccf13cbc828733ed2918817 Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Thu, 14 Apr 2022 11:43:40 +0000 Subject: [PATCH 0049/1954] add parallel decompression, forward arguments and fix compression level --- utils/self-extr-exec/compressor.cpp | 22 +-- utils/self-extr-exec/decompressor.cpp | 212 +++++++++++++++++++------- 2 files changed, 166 insertions(+), 68 deletions(-) diff --git a/utils/self-extr-exec/compressor.cpp b/utils/self-extr-exec/compressor.cpp index 4a067f45890..19e7f0d9848 100644 --- a/utils/self-extr-exec/compressor.cpp +++ b/utils/self-extr-exec/compressor.cpp @@ -25,7 +25,7 @@ Overview of compression: |______________________| */ -/* +/* Metadata contains: 1) number of files to support multiple file compression 2) start_of_files_data to know start of files metadata @@ -147,7 +147,7 @@ int compress(int in_fd, int out_fd, int level, off_t& pointer, const struct stat } /// Shrink file size and unmap - if (0 != ftruncate(out_fd, pointer) || 0 != munmap(input, info_in.st_size) || + if (0 != ftruncate(out_fd, pointer) || 0 != munmap(input, info_in.st_size) || 0 != munmap(output, pointer + info_in.st_size / 3)) { perror(nullptr); @@ -157,7 +157,7 @@ int compress(int in_fd, int out_fd, int level, off_t& pointer, const struct stat } /// Save Metadata at the end of file -int saveMetaData(char* filenames[], int count, int output_fd, const MetaData& metadata, +int saveMetaData(char* filenames[], int count, int output_fd, const MetaData& metadata, FileData* files_data, size_t pointer, size_t sum_file_size) { /// Allocate memory for metadata @@ -168,7 +168,7 @@ int saveMetaData(char* filenames[], int count, int output_fd, const MetaData& me } char * output = static_cast( - mmap(nullptr, + mmap(nullptr, pointer + count * sizeof(FileData) + sum_file_size + sizeof(MetaData), PROT_READ | PROT_WRITE, MAP_SHARED, output_fd, @@ -241,7 +241,8 @@ int compressFiles(char* filenames[], int count, int output_fd, int level, const return 1; } - if (info_in.st_size == 0) { + if (info_in.st_size == 0) + { std::cout << "Empty input file will be skipped." << std::endl; continue; } @@ -252,14 +253,14 @@ int compressFiles(char* filenames[], int count, int output_fd, int level, const /// start of it's compression version files_data[i].uncompressed_size = info_in.st_size; files_data[i].start = pointer; - + /// Compressed data will be added to the end of file /// It will allow to create self extracting executable from file if (0 != compress(input_fd, output_fd, level, pointer, info_in)) { perror(nullptr); delete [] files_data; - return 1; + return 1; } /// This error is less important, than others. @@ -291,14 +292,14 @@ int main(int argc, char* argv[]) std::cout << "Not enough arguments.\ncompressor [OPTIONAL --level of compression] [file name for compressed file] [files that should be compressed]" << std::endl; return 0; } - + int start_of_files = 1; /// Set compression level int level = 5; if (0 == memcmp(argv[1], "--level=", 8)) { - level = strtol(argv[argc - 1], nullptr, 10); + level = strtol(argv[1] + 8, nullptr, 10); ++start_of_files; } @@ -317,10 +318,11 @@ int main(int argc, char* argv[]) return 1; } + std::cout << "Compression with level " << level << std::endl; if (0 != compressFiles(&argv[start_of_files], argc - start_of_files, output_fd, level, info_out)) { std::cout << "Compression was not successful." << std::endl; - + /// Cancel changes. Reset the file to its original state if (0 != ftruncate(output_fd, info_out.st_size)) { diff --git a/utils/self-extr-exec/decompressor.cpp b/utils/self-extr-exec/decompressor.cpp index 34c6e0daad5..706b09d0d71 100644 --- a/utils/self-extr-exec/decompressor.cpp +++ b/utils/self-extr-exec/decompressor.cpp @@ -7,10 +7,11 @@ #include #include #include +#include #include #include -/* +/* Metadata contains: 1) number of files to support multiple file compression 2) start_of_files_data to know start of files metadata @@ -41,19 +42,20 @@ int doDecompress(char * input, char * output, off_t & in_offset, off_t & out_off size_t decompressed_size = ZSTD_decompressDCtx(dctx, output + out_offset, output_size, input + in_offset, input_size); if (ZSTD_isError(decompressed_size)) { + printf("%s\n", ZSTD_getErrorName(decompressed_size)); return 1; } - in_offset += input_size; - out_offset += decompressed_size; return 0; } /// decompress data from in_fd into out_fd -int decompress(char * input, char * output, off_t start, off_t end) +int decompress(char * input, char * output, off_t start, off_t end, size_t max_number_of_forks=10) { off_t in_pointer = start, out_pointer = 0; off_t size = 0; off_t max_block_size = 1ull<<27; + off_t decompressed_size = 0; + size_t number_of_forks = 0; /// Create context ZSTD_DCtx * dctx = ZSTD_createDCtx(); @@ -62,23 +64,89 @@ int decompress(char * input, char * output, off_t start, off_t end) printf("Failed to create context for compression"); return 1; } + pid_t pid; + bool error_happened = false; /// Compress data - while (in_pointer < end) + while (in_pointer < end && !error_happened) { size = ZSTD_findFrameCompressedSize(input + in_pointer, max_block_size); + if (ZSTD_isError(size)) + { + printf("%s\n", ZSTD_getErrorName(size)); + break; + } - /// Compress data or exit if error happens - if (0 != doDecompress(input, output, in_pointer, out_pointer, size, max_block_size, dctx)) - return 1; + decompressed_size = ZSTD_getFrameContentSize(input + in_pointer, max_block_size); + if (ZSTD_isError(decompressed_size)) + { + printf("%s\n", ZSTD_getErrorName(decompressed_size)); + break; + } + + pid = fork(); + if (-1 == pid) + { + perror(nullptr); + /// Decompress data in main process. Exit if error happens + if (0 != doDecompress(input, output, in_pointer, out_pointer, size, max_block_size, dctx)) + break; + } + else if (pid == 0) + { + /// Decompress data. Exit if error happens + if (0 != doDecompress(input, output, in_pointer, out_pointer, size, max_block_size, dctx)) + exit(1); + exit(0); + } + else + { + ++number_of_forks; + while (number_of_forks >= max_number_of_forks) + { + /// Wait any fork + int status; + waitpid(0, &status, 0); + + /// If error happened, stop processing + if (WEXITSTATUS(status) != 0) + { + error_happened = true; + break; + } + + --number_of_forks; + } + in_pointer += size; + out_pointer += decompressed_size; + } } + /// wait for all working decompressions + while (number_of_forks > 0) + { + /// Wait any fork + int status; + waitpid(0, &status, 0); + + if (WEXITSTATUS(status) != 0) + { + error_happened = true; + } + + --number_of_forks; + } + + /// If error happen end of processed part will not reach end + if (in_pointer < end || error_happened) + return 1; + return 0; } /// Read data about files and decomrpess them. -int decompressFiles(int input_fd, char* argv[]) +int decompressFiles(int input_fd, char* argv[], bool & have_compressed_analoge) { /// Read data about output file. /// Compressed data will replace data in file @@ -124,7 +192,7 @@ int decompressFiles(int input_fd, char* argv[]) } if (fs_info.f_blocks * info_in.st_blksize < decompressed_full_size) { - printf("Not enough space for decompression. Have %lu, need %zu.", + printf("Not enough space for decompression. Have %lu, need %zu.", fs_info.f_blocks * info_in.st_blksize, decompressed_full_size); return 1; } @@ -136,7 +204,9 @@ int decompressFiles(int input_fd, char* argv[]) /// Read information about file file_info = *reinterpret_cast(input + files_pointer); files_pointer += sizeof(FileData); - char file_name[file_info.name_length]; + char file_name[file_info.name_length + 1]; + /// Filename should be ended with \0 + memset(file_name, '\0', file_info.name_length + 1); memcpy(file_name, input + files_pointer, file_info.name_length); files_pointer += file_info.name_length; @@ -150,6 +220,7 @@ int decompressFiles(int input_fd, char* argv[]) memcpy(new_file_name, file_name, file_info.name_length); memcpy(new_file_name + file_info.name_length, ".decompressed", 13); output_fd = open(new_file_name, O_RDWR | O_CREAT, 0775); + have_compressed_analoge = true; } else { @@ -173,10 +244,10 @@ int decompressFiles(int input_fd, char* argv[]) } char * output = static_cast( - mmap(nullptr, - file_info.uncompressed_size, - PROT_READ | PROT_WRITE, MAP_SHARED, - output_fd, + mmap(nullptr, + file_info.uncompressed_size, + PROT_READ | PROT_WRITE, MAP_SHARED, + output_fd, 0) ); if (output == MAP_FAILED) @@ -197,25 +268,56 @@ int decompressFiles(int input_fd, char* argv[]) return 1; } - /// TODO: return 1? if (0 != fsync(output_fd)) perror(nullptr); if (0 != close(output_fd)) perror(nullptr); } - + if (0 != munmap(input, info_in.st_size)) perror(nullptr); return 0; } +/// Copy particular part of command and update shift void fill(char * dest, char * source, size_t length, size_t& shift) { memcpy(dest + shift, source, length); shift += length; } -int main(int /*argc*/, char* argv[]) +/// Set command to `mv filename.decompressed filename && filename agrs...` +void fillCommand(char command[], int argc, char * argv[], size_t length) +{ + memset(command, '\0', 3 + strlen(argv[0]) + 14 + strlen(argv[0]) + 4 + strlen(argv[0]) + length + argc); + + /// position in command + size_t shift = 0; + + /// Support variables to create command + char mv[] = "mv "; + char decompressed[] = ".decompressed "; + char add_command[] = " && "; + char space[] = " "; + + fill(command, mv, 3, shift); + fill(command, argv[0], strlen(argv[0]), shift); + fill(command, decompressed, 14, shift); + fill(command, argv[0], strlen(argv[0]), shift); + fill(command, add_command, 4, shift); + fill(command, argv[0], strlen(argv[0]), shift); + fill(command, space, 1, shift); + + /// forward all arguments + for (int i = 1; i < argc; ++i) + { + fill(command, argv[i], strlen(argv[i]), shift); + if (i != argc - 1) + fill(command, space, 1, shift); + } +} + +int main(int argc, char* argv[]) { int input_fd = open(argv[0], O_RDONLY); if (input_fd == -1) @@ -224,8 +326,10 @@ int main(int /*argc*/, char* argv[]) return 0; } + bool have_compressed_analoge = false; + /// Decompress all files - if (0 != decompressFiles(input_fd, argv)) + if (0 != decompressFiles(input_fd, argv, have_compressed_analoge)) { printf("Error happend"); if (0 != close(input_fd)) @@ -242,45 +346,37 @@ int main(int /*argc*/, char* argv[]) /// `rename.ul` is set instead. It will lead to errors /// that can be easily avoided with help of `mv` - // /// TODO: decompressor name can differ from executable - // char bash[] = "/usr/bin/bash"; - // size_t length = 0; - // for (int i = 1; i < argc; ++i) - // length += strlen(argv[i]); - // /// mv filename.decompressed filename && filename agrs... - // char command[8 + 3 + strlen(argv[0]) + 14 + strlen(argv[0]) + 4 + strlen(argv[0]) + length + argc - 1]; - // memset(command, '\0', 8 + 3 + strlen(argv[0]) + 14 + strlen(argv[0]) + 4 + strlen(argv[0]) + length + argc - 1); - - // /// fill command - // size_t shift = 0; - // char executable[] = "bash -c "; - // char mv[] = "mv "; - // char decompressed[] = ".decompressed "; - // char add_command[] = " && "; - // char space[] = " "; - // fill(command, executable, 8, shift); - // fill(command, mv, 3, shift); - // fill(command, argv[0], strlen(argv[0]), shift); - // fill(command, decompressed, 14, shift); - // fill(command, argv[0], strlen(argv[0]), shift); - // fill(command, add_command, 4, shift); - // fill(command, argv[0], strlen(argv[0]), shift); - // fill(command, space, 1, shift); - // for (int i = 1; i < argc; ++i) - // { - // fill(command, argv[i], strlen(argv[i]), shift); - // if (i != argc - 1) - // fill(command, space, 1, shift); - // } - // printf("%s", command); - // fflush(stdout); + if (!have_compressed_analoge) + { + printf("Can't apply arguments to this binary"); + /// remove file + char * name = strrchr(argv[0], '/') + 1; + execlp("rm", "rm", name, NULL); + perror(nullptr); + return 1; + } + else + { + /// move decompressed file instead of this binary and apply command + char bash[] = "/usr/bin/bash"; + char executable[] = "-c"; - // char *newargv[] = { bash, command, nullptr }; - // char *newenviron[] = { nullptr }; - // execve("/usr/bin/bash", newargv, newenviron); + /// length of forwarded args + size_t length = 0; + for (int i = 1; i < argc; ++i) + length += strlen(argv[i]); - /// This part of code will be reached only if error happened - execl("/usr/bin/bash", "bash", "-c", "mv ./clickhouse.decompressed ./clickhouse", NULL); - perror(nullptr); - return 1; + /// mv filename.decompressed filename && filename agrs... + char command[3 + strlen(argv[0]) + 14 + strlen(argv[0]) + 4 + strlen(argv[0]) + length + argc]; + fillCommand(command, argc, argv, length); + + /// replace file and call executable + char * newargv[] = { bash, executable, command, nullptr }; + char * newenviron[] = { nullptr }; + execve("/usr/bin/bash", newargv, newenviron); + + /// This part of code will be reached only if error happened + perror(nullptr); + return 1; + } } From 4ce0b280e623957155a6788affbaaa2ecd70cb33 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 14 Apr 2022 15:31:05 +0200 Subject: [PATCH 0050/1954] wait for node to disappear instead of removing --- src/Common/ZooKeeper/ZooKeeper.cpp | 14 ++++++++++++++ src/Common/ZooKeeper/ZooKeeper.h | 4 ++++ src/Common/randomSeed.cpp | 3 ++- src/Interpreters/DDLWorker.cpp | 10 +--------- .../ReplicatedMergeTreeRestartingThread.cpp | 1 + 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 0f4b141d058..17fcc303081 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -841,6 +841,20 @@ bool ZooKeeper::waitForDisappear(const std::string & path, const WaitCondition & return false; } +void ZooKeeper::waitForEphemeralToDisappearIfAny(const std::string & path) +{ + zkutil::EventPtr eph_node_disappeared = std::make_shared(); + String content; + if (!tryGet(path, content, nullptr, eph_node_disappeared)) + return; + + int32_t timeout_ms = 2 * session_timeout_ms; + if (!eph_node_disappeared->tryWait(timeout_ms)) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, + "Ephemeral node {} still exists after {}s, probably it's owned by someone else. Node data: '{}'", + path, timeout_ms / 1000, content); +} + ZooKeeperPtr ZooKeeper::startNewSession() const { return std::make_shared(hosts, identity, session_timeout_ms, operation_timeout_ms, chroot, implementation, zk_log, get_priority_load_balancing); diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 4d5bd039a55..3a210513e25 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -240,6 +240,10 @@ public: /// The function returns true if waited and false if waiting was interrupted by condition. bool waitForDisappear(const std::string & path, const WaitCondition & condition = {}); + /// Wait for the ephemeral node created in previous session to disappear. + /// Throws LOGICAL_ERROR if node still exists after 2x session_timeout. + void waitForEphemeralToDisappearIfAny(const std::string & path); + /// Async interface (a small subset of operations is implemented). /// /// Usage: diff --git a/src/Common/randomSeed.cpp b/src/Common/randomSeed.cpp index 74088d0a4f3..15a380a1805 100644 --- a/src/Common/randomSeed.cpp +++ b/src/Common/randomSeed.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -29,6 +30,6 @@ DB::UInt64 randomSeed() hash.update(times.tv_nsec); hash.update(times.tv_sec); hash.update(getThreadId()); - hash.update(×); + hash.update(getFQDNOrHostName()); return hash.get64(); } diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 9af6b61a0c1..ff6622949a1 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -545,15 +545,7 @@ void DDLWorker::processTask(DDLTaskBase & task, const ZooKeeperPtr & zookeeper) { /// Connection has been lost and now we are retrying, /// but our previous ephemeral node still exists. - zkutil::EventPtr eph_node_disappeared = std::make_shared(); - String dummy; - if (zookeeper->tryGet(active_node_path, dummy, nullptr, eph_node_disappeared)) - { - constexpr int timeout_ms = 60 * 1000; - if (!eph_node_disappeared->tryWait(timeout_ms)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Ephemeral node {} still exists, " - "probably it's owned by someone else", active_node_path); - } + zookeeper->waitForEphemeralToDisappearIfAny(active_node_path); } zookeeper->create(active_node_path, {}, zkutil::CreateMode::Ephemeral); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 9b4234a99e5..7df1c0a13d8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -303,6 +303,7 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() ReplicatedMergeTreeAddress address = storage.getReplicatedMergeTreeAddress(); String is_active_path = fs::path(storage.replica_path) / "is_active"; + zookeeper->waitForEphemeralToDisappearIfAny(is_active_path); /// Simultaneously declare that this replica is active, and update the host. Coordination::Requests ops; From 29e0b4c3bb3f5469ca171d1b4cfad7ca75c39ba0 Mon Sep 17 00:00:00 2001 From: yuuch Date: Fri, 15 Apr 2022 00:12:31 +0800 Subject: [PATCH 0051/1954] support collation in create query --- src/Core/Settings.h | 2 ++ src/Interpreters/InterpreterCreateQuery.cpp | 6 ++++- src/Parsers/ASTCollation.cpp | 27 +++++++++++++++++++ src/Parsers/ASTCollation.h | 22 +++++++++++++++ src/Parsers/ASTColumnDeclaration.cpp | 11 ++++++++ src/Parsers/ASTColumnDeclaration.h | 1 + src/Parsers/ExpressionElementParsers.cpp | 27 +++++++++++++++++++ src/Parsers/ExpressionElementParsers.h | 10 +++++++ src/Parsers/ParserCreateQuery.h | 13 +++++++++ ...2267_create_table_with_collation.reference | 0 .../02267_create_table_with_collation.sql | 7 +++++ 11 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 src/Parsers/ASTCollation.cpp create mode 100644 src/Parsers/ASTCollation.h create mode 100644 tests/queries/0_stateless/02267_create_table_with_collation.reference create mode 100644 tests/queries/0_stateless/02267_create_table_with_collation.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index aa78456702c..996c80cbdf3 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -351,6 +351,8 @@ class IColumn; M(UInt64, partial_merge_join_left_table_buffer_bytes, 0, "If not 0 group left table blocks in bigger ones for left-side table in partial merge join. It uses up to 2x of specified memory per joining thread.", 0) \ M(UInt64, partial_merge_join_rows_in_right_blocks, 65536, "Split right-hand joining data in blocks of specified size. It's a portion of data indexed by min-max values and possibly unloaded on disk.", 0) \ M(UInt64, join_on_disk_max_files_to_merge, 64, "For MergeJoin on disk set how much files it's allowed to sort simultaneously. Then this value bigger then more memory used and then less disk I/O needed. Minimum is 2.", 0) \ + M(Bool, compatibility_ignore_collation_in_create_table, true, "compatibility ignore collation in create table", 0) \ + \ M(String, temporary_files_codec, "LZ4", "Set compression codec for temporary files (sort and join on disk). I.e. LZ4, NONE.", 0) \ \ M(UInt64, max_rows_to_transfer, 0, "Maximum size (in rows) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 24c58c819a4..008fde7e805 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -460,8 +460,12 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( { const auto & col_decl = ast->as(); - DataTypePtr column_type = nullptr; + if (col_decl.collation && !context_->getSettingsRef().compatibility_ignore_collation_in_create_table) + { + throw Exception{"Cannot support collation, please set compatibility_ignore_collation_in_create_table=true", ErrorCodes::LOGICAL_ERROR}; + } + DataTypePtr column_type = nullptr; if (col_decl.type) { column_type = DataTypeFactory::instance().get(col_decl.type); diff --git a/src/Parsers/ASTCollation.cpp b/src/Parsers/ASTCollation.cpp new file mode 100644 index 00000000000..c41eff16128 --- /dev/null +++ b/src/Parsers/ASTCollation.cpp @@ -0,0 +1,27 @@ +#include + +namespace DB +{ + ASTPtr ASTCollation::clone() const + { + auto res = std::make_shared(*this); + res->null_modifier = null_modifier; + res->collation = collation; + return res; + } + + void ASTCollation::formatImpl(const FormatSettings &s, FormatState &state, FormatStateStacked frame) const + { + if (collation) + { + collation->formatImpl(s, state, frame); + } + + if (!null_modifier) + { + s.ostr << " NOT NULL "; + } + + } + +} diff --git a/src/Parsers/ASTCollation.h b/src/Parsers/ASTCollation.h new file mode 100644 index 00000000000..042a7f4e0e7 --- /dev/null +++ b/src/Parsers/ASTCollation.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +namespace DB +{ + +class ASTCollation : public IAST +{ +public: + ASTPtr collation = nullptr; + std::optional null_modifier; + + + String getID(char) const override { return "Collation"; } + + ASTPtr clone() const override; + + void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; +}; + +} diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index 888cd639fb0..e5334bb00fb 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -43,6 +43,11 @@ ASTPtr ASTColumnDeclaration::clone() const res->ttl = ttl->clone(); res->children.push_back(res->ttl); } + if (collation) + { + res->collation = collation->clone(); + res->children.push_back(res->collation); + } return res; } @@ -97,6 +102,12 @@ void ASTColumnDeclaration::formatImpl(const FormatSettings & settings, FormatSta settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "TTL" << (settings.hilite ? hilite_none : "") << ' '; ttl->formatImpl(settings, state, frame); } + + if (collation) + { + settings.ostr << ' ' << (settings.hilite ? hilite_keyword : "") << "COLLATION" << (settings.hilite ? hilite_none : "") << ' '; + collation->formatImpl(settings, state, frame); + } } } diff --git a/src/Parsers/ASTColumnDeclaration.h b/src/Parsers/ASTColumnDeclaration.h index ea17a8b4dfa..5ecfb859abc 100644 --- a/src/Parsers/ASTColumnDeclaration.h +++ b/src/Parsers/ASTColumnDeclaration.h @@ -19,6 +19,7 @@ public: ASTPtr comment; ASTPtr codec; ASTPtr ttl; + ASTPtr collation; String getID(char delim) const override { return "ColumnDeclaration" + (delim + name); } diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index cd07e304a39..bb2b80eb20d 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -1309,6 +1310,32 @@ bool ParserCodec::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return true; } +bool ParserCollation::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword s_null{"NULL"}; + ParserKeyword s_not{"NOT"}; + std::optional null_modifier; + ParserLiteral literal_p; + + ASTPtr collation; + + if (!ParserIdentifier(true).parse(pos, collation, expected)) + return false; + if (s_not.ignore(pos, expected)) + { + if (!s_null.ignore(pos, expected)) + return false; + null_modifier.emplace(false); + } + else if (s_null.ignore(pos, expected)) + null_modifier.emplace(true); + auto collation_node = std::make_shared(); + collation_node->collation = collation; + collation_node->null_modifier = null_modifier; + node = collation_node; + return true; +} + template static bool isOneOf(TokenType token) diff --git a/src/Parsers/ExpressionElementParsers.h b/src/Parsers/ExpressionElementParsers.h index 10b4f5fd7d1..e823c7bf46f 100644 --- a/src/Parsers/ExpressionElementParsers.h +++ b/src/Parsers/ExpressionElementParsers.h @@ -217,6 +217,16 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +/** Parse collation + * COLLATE utf8_unicode_ci NOT NULL + */ +class ParserCollation : public IParserBase +{ +protected: + const char * getName() const override { return "collation"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + /// Fast path of cast operator "::". /// It tries to read literal as text. /// If it fails, later operator will be transformed to function CAST. diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index a6d3476e3f7..702c8402f6a 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -132,9 +132,11 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E ParserKeyword s_ttl{"TTL"}; ParserKeyword s_remove{"REMOVE"}; ParserKeyword s_type{"TYPE"}; + ParserKeyword s_collate{"COLLATE"}; ParserTernaryOperatorExpression expr_parser; ParserStringLiteral string_literal_parser; ParserCodec codec_parser; + ParserCollation collation_parser; ParserExpression expression_parser; /// mandatory column name @@ -170,6 +172,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E ASTPtr comment_expression; ASTPtr codec_expression; ASTPtr ttl_expression; + ASTPtr collation_expression; if (!s_default.checkWithoutMoving(pos, expected) && !s_materialized.checkWithoutMoving(pos, expected) @@ -183,6 +186,11 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E return false; if (!type_parser.parse(pos, type, expected)) return false; + if (s_collate.ignore(pos, expected)) + { + if (!collation_parser.parse(pos, collation_expression, expected)) + return false; + } } Pos pos_before_specifier = pos; @@ -269,6 +277,11 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E column_declaration->ttl = ttl_expression; column_declaration->children.push_back(std::move(ttl_expression)); } + if (collation_expression) + { + column_declaration->collation = collation_expression; + column_declaration->children.push_back(std::move(collation_expression)); + } return true; } diff --git a/tests/queries/0_stateless/02267_create_table_with_collation.reference b/tests/queries/0_stateless/02267_create_table_with_collation.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02267_create_table_with_collation.sql b/tests/queries/0_stateless/02267_create_table_with_collation.sql new file mode 100644 index 00000000000..bf979e6e3d0 --- /dev/null +++ b/tests/queries/0_stateless/02267_create_table_with_collation.sql @@ -0,0 +1,7 @@ +SET compatibility_ignore_collation_in_create_table=false; +CREATE TABLE t_02267_collation (x varchar(255) COLLATION utf8_unicode_ci NOT NULL) ENGINE = Memory; --{serverError 49} + +SET compatibility_ignore_collation_in_create_table=false; +CREATE TABLE t_02267_collation (x varchar(255) COLLATION utf8_unicode_ci NOT NULL) ENGINE = Memory; + +DROP TABLE t_02267_collation; From 0a3d51a162816d0bbeb3892e07655d7be29cf3bc Mon Sep 17 00:00:00 2001 From: yuuch Date: Fri, 15 Apr 2022 01:17:48 +0800 Subject: [PATCH 0052/1954] try restart CI From 6919ca9a6e31225e51ce3366c36f1d4f3ec4acec Mon Sep 17 00:00:00 2001 From: yuuch Date: Fri, 15 Apr 2022 21:31:25 +0800 Subject: [PATCH 0053/1954] remove parse about the not null in ParserCollation --- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- src/Parsers/ASTCollation.cpp | 6 ------ src/Parsers/ASTCollation.h | 2 -- src/Parsers/ExpressionElementParsers.cpp | 14 -------------- 4 files changed, 1 insertion(+), 23 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 008fde7e805..be250bc34c5 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -462,7 +462,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( if (col_decl.collation && !context_->getSettingsRef().compatibility_ignore_collation_in_create_table) { - throw Exception{"Cannot support collation, please set compatibility_ignore_collation_in_create_table=true", ErrorCodes::LOGICAL_ERROR}; + throw Exception("Cannot support collation, please set compatibility_ignore_collation_in_create_table=true", ErrorCodes::LOGICAL_ERROR); } DataTypePtr column_type = nullptr; diff --git a/src/Parsers/ASTCollation.cpp b/src/Parsers/ASTCollation.cpp index c41eff16128..74944e20959 100644 --- a/src/Parsers/ASTCollation.cpp +++ b/src/Parsers/ASTCollation.cpp @@ -5,7 +5,6 @@ namespace DB ASTPtr ASTCollation::clone() const { auto res = std::make_shared(*this); - res->null_modifier = null_modifier; res->collation = collation; return res; } @@ -17,11 +16,6 @@ namespace DB collation->formatImpl(s, state, frame); } - if (!null_modifier) - { - s.ostr << " NOT NULL "; - } - } } diff --git a/src/Parsers/ASTCollation.h b/src/Parsers/ASTCollation.h index 042a7f4e0e7..a735956a90e 100644 --- a/src/Parsers/ASTCollation.h +++ b/src/Parsers/ASTCollation.h @@ -9,8 +9,6 @@ class ASTCollation : public IAST { public: ASTPtr collation = nullptr; - std::optional null_modifier; - String getID(char) const override { return "Collation"; } diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index bb2b80eb20d..89245478e6a 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -1312,26 +1312,12 @@ bool ParserCodec::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) bool ParserCollation::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - ParserKeyword s_null{"NULL"}; - ParserKeyword s_not{"NOT"}; - std::optional null_modifier; - ParserLiteral literal_p; - ASTPtr collation; if (!ParserIdentifier(true).parse(pos, collation, expected)) return false; - if (s_not.ignore(pos, expected)) - { - if (!s_null.ignore(pos, expected)) - return false; - null_modifier.emplace(false); - } - else if (s_null.ignore(pos, expected)) - null_modifier.emplace(true); auto collation_node = std::make_shared(); collation_node->collation = collation; - collation_node->null_modifier = null_modifier; node = collation_node; return true; } From 61b5010319b0ea1e41f117912724335b6a0ad5a4 Mon Sep 17 00:00:00 2001 From: yuuch Date: Sat, 16 Apr 2022 01:01:44 +0800 Subject: [PATCH 0054/1954] try restart ci again. From 8daebf38b91c336227460c4f83cae3e6fbfb6a37 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 16 Apr 2022 00:48:10 +0200 Subject: [PATCH 0055/1954] Remove "google-readability-casting" because constructor-style cast is Ok --- .clang-tidy | 1 - 1 file changed, 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 6fd67876923..3ffc6026f0f 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -118,7 +118,6 @@ Checks: '-*, google-build-namespaces, google-default-arguments, google-explicit-constructor, - google-readability-casting, google-readability-avoid-underscore-in-googletest-name, google-runtime-int, google-runtime-operator, From de7725083e6e7a3d3c1614ff2e426ff9411cd9e8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 16 Apr 2022 01:15:40 +0200 Subject: [PATCH 0056/1954] Fix clang-tidy, part 1 --- contrib/sysroot | 2 +- src/Access/DiskAccessStorage.cpp | 3 ++- .../AggregateFunctionAvgWeighted.cpp | 4 ++-- src/AggregateFunctions/AggregateFunctionDeltaSum.cpp | 2 +- .../AggregateFunctionGroupArray.cpp | 2 +- .../AggregateFunctionGroupBitmap.cpp | 2 +- .../AggregateFunctionGroupUniqArray.cpp | 2 +- src/AggregateFunctions/AggregateFunctionQuantile.cpp | 4 ++-- src/AggregateFunctions/AggregateFunctionSparkbar.cpp | 2 +- .../AggregateFunctionStatisticsSimple.cpp | 2 +- src/AggregateFunctions/AggregateFunctionSum.cpp | 2 +- src/AggregateFunctions/AggregateFunctionSumCount.cpp | 2 +- src/AggregateFunctions/AggregateFunctionTopK.cpp | 2 +- src/Common/UTF8Helpers.cpp | 2 +- src/Dictionaries/IPAddressDictionary.cpp | 12 ++++++------ src/Functions/FunctionHelpers.cpp | 2 +- src/Functions/FunctionsLogical.cpp | 6 +++--- src/Functions/array/arrayElement.cpp | 2 +- src/Functions/if.cpp | 6 +++--- src/Functions/toLowCardinality.cpp | 1 - src/Interpreters/JIT/compileFunction.cpp | 2 +- src/Interpreters/convertFieldToType.cpp | 12 ++++++------ src/Parsers/Access/ASTShowAccessEntitiesQuery.cpp | 3 ++- src/Storages/FileLog/StorageFileLog.cpp | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- src/Storages/MergeTree/MergeTreeDataWriter.cpp | 2 +- src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp | 2 +- .../MergeTree/MergeTreeIndexGranuleBloomFilter.cpp | 2 +- src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp | 2 +- 29 files changed, 46 insertions(+), 45 deletions(-) diff --git a/contrib/sysroot b/contrib/sysroot index e9fb375d0a1..bbcac834526 160000 --- a/contrib/sysroot +++ b/contrib/sysroot @@ -1 +1 @@ -Subproject commit e9fb375d0a1e5ebfd74c043f088f2342552103f8 +Subproject commit bbcac834526d90d1e764164b861be426891d1743 diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 7393fcd8d36..a9eb27c291c 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -327,7 +327,8 @@ void DiskAccessStorage::scheduleWriteLists(AccessEntityType type) /// Create the 'need_rebuild_lists.mark' file. /// This file will be used later to find out if writing lists is successful or not. - std::ofstream{getNeedRebuildListsMarkFilePath(directory_path)}; + std::ofstream out{getNeedRebuildListsMarkFilePath(directory_path)}; + out.close(); lists_writing_thread = ThreadFromGlobalPool{&DiskAccessStorage::listsWritingThreadFunc, this}; lists_writing_thread_is_waiting = true; diff --git a/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp b/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp index ab6fdc8fd7e..4d7901a7fac 100644 --- a/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp +++ b/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp @@ -39,7 +39,7 @@ bool allowTypes(const DataTypePtr& left, const DataTypePtr& right) noexcept } template -static IAggregateFunction * create(const IDataType & second_type, TArgs && ... args) +IAggregateFunction * create(const IDataType & second_type, TArgs && ... args) { const WhichDataType which(second_type); @@ -51,7 +51,7 @@ static IAggregateFunction * create(const IDataType & second_type, TArgs && ... a // Not using helper functions because there are no templates for binary decimal/numeric function. template -static IAggregateFunction * create(const IDataType & first_type, const IDataType & second_type, TArgs && ... args) +IAggregateFunction * create(const IDataType & first_type, const IDataType & second_type, TArgs && ... args) { const WhichDataType which(first_type); diff --git a/src/AggregateFunctions/AggregateFunctionDeltaSum.cpp b/src/AggregateFunctions/AggregateFunctionDeltaSum.cpp index f1c6e7c6112..3b43d9a85f8 100644 --- a/src/AggregateFunctions/AggregateFunctionDeltaSum.cpp +++ b/src/AggregateFunctions/AggregateFunctionDeltaSum.cpp @@ -30,7 +30,7 @@ AggregateFunctionPtr createAggregateFunctionDeltaSum( throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - DataTypePtr data_type = arguments[0]; + const DataTypePtr & data_type = arguments[0]; if (isInteger(data_type) || isFloat(data_type)) return AggregateFunctionPtr(createWithNumericType( diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp index 5a9fd778277..85075d5a4d6 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp @@ -20,7 +20,7 @@ namespace { template